Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/../logs/demand-backup.log
+ create_infra demand-backup-7490
+ local ns=demand-backup-7490
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.EQRnmO1FNZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.7cLiVarFUH
+ local exit_status=0
+ local timeout=4
+ args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml' '--ignore-not-found' '--wait=false')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EQRnmO1FNZ
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.7cLiVarFUH
+ rm /tmp/tmp.EQRnmO1FNZ /tmp/tmp.7cLiVarFUH
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-28675 backup-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-28675 backup-oss-native --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-oss-native patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.c7cmXQYJ5Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.69DlJxhTYO
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.c7cmXQYJ5Y
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.69DlJxhTYO
+ rm /tmp/tmp.c7cmXQYJ5Y /tmp/tmp.69DlJxhTYO
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.uhcTHy2Nn4
++ mktemp
+ local LAST_ERR=/tmp/tmp.ukq1BPww6V
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uhcTHy2Nn4
+ cat /tmp/tmp.ukq1BPww6V
+ rm /tmp/tmp.uhcTHy2Nn4 /tmp/tmp.ukq1BPww6V
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.naHy9pUac5
++ mktemp
+ local LAST_ERR=/tmp/tmp.E1bNegwnY1
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.naHy9pUac5
+ cat /tmp/tmp.E1bNegwnY1
+ rm /tmp/tmp.naHy9pUac5 /tmp/tmp.E1bNegwnY1
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.21IsMlswNU
++ mktemp
+ local LAST_ERR=/tmp/tmp.XPcTDPUE5U
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.21IsMlswNU
+ cat /tmp/tmp.XPcTDPUE5U
+ rm /tmp/tmp.21IsMlswNU /tmp/tmp.XPcTDPUE5U
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.bDdo3FTnDG
++ mktemp
+ local LAST_ERR=/tmp/tmp.3srYssNzqz
+ local exit_status=0
+ local timeout=4
+ args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bDdo3FTnDG
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.3srYssNzqz
+ rm /tmp/tmp.bDdo3FTnDG /tmp/tmp.3srYssNzqz
+ return 0
+ check_crd_for_deletion PR-2474-67718aa29
+ local git_tag=PR-2474-67718aa29
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2474-67718aa29/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.81Waeinuji
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F9mO8SCipb
++ local exit_status=0
++ local timeout=4
++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.81Waeinuji
++ cat /tmp/tmp.F9mO8SCipb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.81Waeinuji
++ cat /tmp/tmp.F9mO8SCipb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.81Waeinuji
++ cat /tmp/tmp.F9mO8SCipb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.81Waeinuji
++ cat /tmp/tmp.F9mO8SCipb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.81Waeinuji /tmp/tmp.F9mO8SCipb
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ [[ -n '' ]]
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.t3TN0B45XN
+ local LAST_OUT=/tmp/tmp.CXOA84XrTZ
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.vIRZzaMQA8
+ local exit_status=0
+ local timeout=4
+ args=('get' 'ns')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.gjmUOnFR6A
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t3TN0B45XN
+ cat /tmp/tmp.vIRZzaMQA8
+ rm /tmp/tmp.t3TN0B45XN /tmp/tmp.vIRZzaMQA8
+ return 0
namespace "demand-backup-28675" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CXOA84XrTZ
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.gjmUOnFR6A
+ rm /tmp/tmp.CXOA84XrTZ /tmp/tmp.gjmUOnFR6A
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.EViK2zSIGf
++ mktemp
+ local LAST_ERR=/tmp/tmp.mJHUHCwp1l
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'namespace' 'psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EViK2zSIGf
+ cat /tmp/tmp.mJHUHCwp1l
+ rm /tmp/tmp.EViK2zSIGf /tmp/tmp.mJHUHCwp1l
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.IgvMzBmQLt
++ mktemp
+ local LAST_ERR=/tmp/tmp.h46BfYm9Yi
+ local exit_status=0
+ local timeout=4
+ args=('create' 'namespace' 'psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IgvMzBmQLt
namespace/psmdb-operator created
+ cat /tmp/tmp.h46BfYm9Yi
+ rm /tmp/tmp.IgvMzBmQLt /tmp/tmp.h46BfYm9Yi
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jfklvd2PTo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K90ljSELCs
++ local exit_status=0
++ local timeout=4
++ args=('config' 'current-context')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jfklvd2PTo
++ cat /tmp/tmp.K90ljSELCs
++ rm /tmp/tmp.Jfklvd2PTo /tmp/tmp.K90ljSELCs
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.w5rn2f2e1T
++ mktemp
+ local LAST_ERR=/tmp/tmp.nKUubVeSJI
+ local exit_status=0
+ local timeout=4
+ args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12' '--namespace=psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.w5rn2f2e1T
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12" modified.
+ cat /tmp/tmp.nKUubVeSJI
+ rm /tmp/tmp.w5rn2f2e1T /tmp/tmp.nKUubVeSJI
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ngDJdNUlkl
++ mktemp
+ local LAST_ERR=/tmp/tmp.8aLrAxVDMM
+ local exit_status=0
+ local timeout=4
+ args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ngDJdNUlkl
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.8aLrAxVDMM
+ rm /tmp/tmp.ngDJdNUlkl /tmp/tmp.8aLrAxVDMM
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.I5wd04IZmv
++ mktemp
+ local LAST_ERR=/tmp/tmp.5l4kOWPAu8
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-n' 'psmdb-operator' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I5wd04IZmv
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.5l4kOWPAu8
+ rm /tmp/tmp.I5wd04IZmv /tmp/tmp.5l4kOWPAu8
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
+ add_arch_to_podspecs -
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ cat -
++ mktemp
+ local LAST_OUT=/tmp/tmp.QoLNDbd5wB
++ mktemp
+ local LAST_ERR=/tmp/tmp.T7eqQBZPkv
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-n' 'psmdb-operator' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ return
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QoLNDbd5wB
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.T7eqQBZPkv
+ rm /tmp/tmp.QoLNDbd5wB /tmp/tmp.T7eqQBZPkv
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IdqAHMOmox
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wW34y2sagj
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IdqAHMOmox
++ cat /tmp/tmp.wW34y2sagj
++ rm /tmp/tmp.IdqAHMOmox /tmp/tmp.wW34y2sagj
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-555df8f55f-jmlhs
+ local pod=percona-server-mongodb-operator-555df8f55f-jmlhs
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-555df8f55f-jmlhs to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2KuwEVuLkw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z8vCGWCy8c
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2KuwEVuLkw
++ cat /tmp/tmp.z8vCGWCy8c
++ rm /tmp/tmp.2KuwEVuLkw /tmp/tmp.z8vCGWCy8c
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-555df8f55f-jmlhs
++ mktemp
+ local LAST_OUT=/tmp/tmp.JZTNZaCeLT
++ mktemp
+ local LAST_ERR=/tmp/tmp.xwsJPjXypL
+ local exit_status=0
+ local timeout=4
+ args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-555df8f55f-jmlhs')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-555df8f55f-jmlhs
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JZTNZaCeLT
+ cat /tmp/tmp.xwsJPjXypL
+ rm /tmp/tmp.JZTNZaCeLT /tmp/tmp.xwsJPjXypL
+ return 0
2026-07-22T13:24:21.398Z	INFO	setup	Manager starting up	{"gitCommit": "67718aa293399ea519feb6abcdfab18192ca7f41", "gitBranch": "PR-2474-67718aa29", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-7490
+ local namespace=demand-backup-7490
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ [[ -n '' ]]
+ desc 'cleaned up old namespaces demand-backup-7490'
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-7490
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-7490 --ignore-not-found
+ awk '{print$1}'
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.WbiqKzfheQ
+ local LAST_OUT=/tmp/tmp.cQnWOoUNdg
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.eroFQazseV
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'namespace' 'demand-backup-7490' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ local LAST_ERR=/tmp/tmp.cZQYr7CzwS
+ local exit_status=0
+ local timeout=4
+ args=('get' 'ns')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
++ seq 0 2
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-7490 --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cQnWOoUNdg
+ cat /tmp/tmp.cZQYr7CzwS
+ rm /tmp/tmp.cQnWOoUNdg /tmp/tmp.cZQYr7CzwS
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WbiqKzfheQ
+ cat /tmp/tmp.eroFQazseV
+ rm /tmp/tmp.WbiqKzfheQ /tmp/tmp.eroFQazseV
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-7490
++ mktemp
+ local LAST_OUT=/tmp/tmp.jOBeTq6VHE
++ mktemp
+ local LAST_ERR=/tmp/tmp.xsU8rtcwlm
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'namespace' 'demand-backup-7490')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-7490
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jOBeTq6VHE
+ cat /tmp/tmp.xsU8rtcwlm
+ rm /tmp/tmp.jOBeTq6VHE /tmp/tmp.xsU8rtcwlm
+ return 0
+ desc 'create namespace demand-backup-7490'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-7490
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-7490
++ mktemp
+ local LAST_OUT=/tmp/tmp.rcmca3gPmx
++ mktemp
+ local LAST_ERR=/tmp/tmp.8DoMyxE7fz
+ local exit_status=0
+ local timeout=4
+ args=('create' 'namespace' 'demand-backup-7490')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-7490
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rcmca3gPmx
namespace/demand-backup-7490 created
+ cat /tmp/tmp.8DoMyxE7fz
+ rm /tmp/tmp.rcmca3gPmx /tmp/tmp.8DoMyxE7fz
+ return 0
+ set_kube_ctx demand-backup-7490
+ local namespace=demand-backup-7490
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.04qCMD6bEK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZgcRfCgGqy
++ local exit_status=0
++ local timeout=4
++ args=('config' 'current-context')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.04qCMD6bEK
++ cat /tmp/tmp.ZgcRfCgGqy
++ rm /tmp/tmp.04qCMD6bEK /tmp/tmp.ZgcRfCgGqy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12 --namespace=demand-backup-7490
++ mktemp
+ local LAST_OUT=/tmp/tmp.kjHYu98EyC
++ mktemp
+ local LAST_ERR=/tmp/tmp.9LdelIxEYo
+ local exit_status=0
+ local timeout=4
+ args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12' '--namespace=demand-backup-7490')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12 --namespace=demand-backup-7490
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kjHYu98EyC
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster12" modified.
+ cat /tmp/tmp.9LdelIxEYo
+ rm /tmp/tmp.kjHYu98EyC /tmp/tmp.9LdelIxEYo
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Jul 22 13:25:00 2026
NAMESPACE: demand-backup-7490
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-7490.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-7490 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-7490

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-7490 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-7490 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dpnbAdlpSm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bkG08WRLdG
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=release=minio-service' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dpnbAdlpSm
++ cat /tmp/tmp.bkG08WRLdG
++ rm /tmp/tmp.dpnbAdlpSm /tmp/tmp.bkG08WRLdG
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-qvcwq
+ wait_pod minio-service-6d5f646cdc-qvcwq
+ local pod=minio-service-6d5f646cdc-qvcwq
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-qvcwq to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-7490.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ aws_cli_minio http://minio-service:9000 s3 mb s3://operator-testing
+ local endpoint=http://minio-service:9000
+ shift
+ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.2FJVKI98uw
++ mktemp
+ local LAST_ERR=/tmp/tmp.wOgvmCIIGv
+ local exit_status=0
+ local timeout=4
+ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'mb' 's3://operator-testing')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2FJVKI98uw
pod "aws-cli" deleted from demand-backup-7490 namespace
+ cat /tmp/tmp.wOgvmCIIGv
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.2FJVKI98uw /tmp/tmp.wOgvmCIIGv
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ cluster=some-name-rs0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.06oFYpYeG3
++ mktemp
+ local LAST_ERR=/tmp/tmp.lGMPWagM4m
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.06oFYpYeG3
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.lGMPWagM4m
+ rm /tmp/tmp.06oFYpYeG3 /tmp/tmp.lGMPWagM4m
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.d1LYKQ1yAE
++ mktemp
+ local LAST_ERR=/tmp/tmp.lhmb08ntpP
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d1LYKQ1yAE
secret/minio-secret created
secret/aws-s3-secret created
secret/alibaba-cloud-secret created
secret/oci-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.lhmb08ntpP
+ rm /tmp/tmp.d1LYKQ1yAE /tmp/tmp.lhmb08ntpP
+ return 0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.YnMTmr0v8i
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ [[ -n docker.io/perconalab/fluentbit:main-logcollector ]]
+ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector
+ yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)'
++ mktemp
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-7490/g
+ yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"'
+ add_arch_to_psmdb -
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ cat -
+ local LAST_ERR=/tmp/tmp.MLy3x21hxF
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ return
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YnMTmr0v8i
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.MLy3x21hxF
+ rm /tmp/tmp.YnMTmr0v8i /tmp/tmp.MLy3x21hxF
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready...............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5lJMvZvuG4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2cOi5vqp6C
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5lJMvZvuG4
++ cat /tmp/tmp.2cOi5vqp6C
++ rm /tmp/tmp.5lJMvZvuG4 /tmp/tmp.2cOi5vqp6C
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mqkcWLYWQF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WrDRGOcL4l
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mqkcWLYWQF
++ cat /tmp/tmp.WrDRGOcL4l
++ rm /tmp/tmp.mqkcWLYWQF /tmp/tmp.WrDRGOcL4l
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.glUXtLjbfH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.axYVoDLi3E
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.glUXtLjbfH
++ cat /tmp/tmp.axYVoDLi3E
++ rm /tmp/tmp.glUXtLjbfH /tmp/tmp.axYVoDLi3E
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster some-name to be ready.................
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.QVjC8nwngm/statefulset_some-name-rs0.yml
+ [[ -n '' ]]
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-7490", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.uO3JXN4a98
++ mktemp
+ local LAST_ERR=/tmp/tmp.zp0DfNMlA9
+ local exit_status=0
+ local timeout=4
+ args=('get' '-o' 'yaml' 'statefulset/some-name-rs0')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uO3JXN4a98
+ cat /tmp/tmp.zp0DfNMlA9
+ rm /tmp/tmp.uO3JXN4a98 /tmp/tmp.zp0DfNMlA9
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.QVjC8nwngm/statefulset_some-name-rs0.yml
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.QVjC8nwngm/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.QVjC8nwngm/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.QVjC8nwngm/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-07-22T13:28:15+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ check_custom_backup_probes some-name-rs0
+ local cluster=some-name-rs0
+ desc 'verify custom backup-agent liveness/readiness probes set via CR are applied'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify custom backup-agent liveness/readiness probes set via CR are applied
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t  {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t  {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.G92ZOpzFZN
++ mktemp
+ local LAST_ERR=/tmp/tmp.q3vW6tm7m6
+ local exit_status=0
+ local timeout=4
+ args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[
	  {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},
	  {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}
	]')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=json $'-p=[\n\t  {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t  {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G92ZOpzFZN
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.q3vW6tm7m6
+ rm /tmp/tmp.G92ZOpzFZN /tmp/tmp.q3vW6tm7m6
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W25GIvmBaT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WsohCuaIaP
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W25GIvmBaT
++ cat /tmp/tmp.WsohCuaIaP
++ rm /tmp/tmp.W25GIvmBaT /tmp/tmp.WsohCuaIaP
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xl06gB3pjM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wDpTAI7k6q
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xl06gB3pjM
++ cat /tmp/tmp.wDpTAI7k6q
++ rm /tmp/tmp.Xl06gB3pjM /tmp/tmp.wDpTAI7k6q
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s5kvaE3dJy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pkGa5y17qn
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s5kvaE3dJy
++ cat /tmp/tmp.pkGa5y17qn
++ rm /tmp/tmp.s5kvaE3dJy /tmp/tmp.pkGa5y17qn
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster some-name to be ready................................................
+ local liveness readiness
++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H2d1rT8VCo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PAtSoaVuPZ
++ local exit_status=0
++ local timeout=4
++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H2d1rT8VCo
++ cat /tmp/tmp.PAtSoaVuPZ
++ rm /tmp/tmp.H2d1rT8VCo /tmp/tmp.PAtSoaVuPZ
++ return 0
+ liveness=/bin/true
++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FvbVlWf3kn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GeUXhmCxbc
++ local exit_status=0
++ local timeout=4
++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FvbVlWf3kn
++ cat /tmp/tmp.GeUXhmCxbc
++ rm /tmp/tmp.FvbVlWf3kn /tmp/tmp.GeUXhmCxbc
++ return 0
+ readiness=/bin/true
+ [[ /bin/true != /bin/true ]]
+ [[ /bin/true != /bin/true ]]
+ desc 'revert custom backup-agent probes to defaults'
+ set +o xtrace


-----------------------------------------------------------------------------------
revert custom backup-agent probes to defaults
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t  {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t  {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.il5ds2XPCI
++ mktemp
+ local LAST_ERR=/tmp/tmp.rIF2nUpTlz
+ local exit_status=0
+ local timeout=4
+ args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[
	  {"op":"remove","path":"/spec/backup/livenessProbe"},
	  {"op":"remove","path":"/spec/backup/readinessProbe"}
	]')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=json $'-p=[\n\t  {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t  {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.il5ds2XPCI
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.rIF2nUpTlz
+ rm /tmp/tmp.il5ds2XPCI /tmp/tmp.rIF2nUpTlz
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3aBZE5SbFc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RQd18yLfIM
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3aBZE5SbFc
++ cat /tmp/tmp.RQd18yLfIM
++ rm /tmp/tmp.3aBZE5SbFc /tmp/tmp.RQd18yLfIM
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qT5hW5jXqH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nL7573qycE
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qT5hW5jXqH
++ cat /tmp/tmp.nL7573qycE
++ rm /tmp/tmp.qT5hW5jXqH /tmp/tmp.nL7573qycE
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JzLqcgZabL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xABgrLCP4m
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JzLqcgZabL
++ cat /tmp/tmp.xABgrLCP4m
++ rm /tmp/tmp.JzLqcgZabL /tmp/tmp.xABgrLCP4m
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster some-name to be ready......................................
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VH2GZlR4EW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J6bppsTSdI
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VH2GZlR4EW
++ cat /tmp/tmp.J6bppsTSdI
++ rm /tmp/tmp.VH2GZlR4EW /tmp/tmp.J6bppsTSdI
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.E3M76mD4jt
++ mktemp
+ local LAST_ERR=/tmp/tmp.qSjn9QJTly
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E3M76mD4jt
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c7624cb7-2861-4b95-a660-bf088d526150") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.qSjn9QJTly
+ rm /tmp/tmp.E3M76mD4jt /tmp/tmp.qSjn9QJTly
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gqAW42VAcB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RNGsSTqcmw
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gqAW42VAcB
++ cat /tmp/tmp.RNGsSTqcmw
++ rm /tmp/tmp.gqAW42VAcB /tmp/tmp.RNGsSTqcmw
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uBgfdensYD
++ mktemp
+ local LAST_ERR=/tmp/tmp.NBEAmDmXhN
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uBgfdensYD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1d7b7143-671d-490f-9a22-48500e3a3557") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NBEAmDmXhN
+ rm /tmp/tmp.uBgfdensYD /tmp/tmp.NBEAmDmXhN
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8UzsnYi7xn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3S8gzEtUF7
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8UzsnYi7xn
++ cat /tmp/tmp.3S8gzEtUF7
++ rm /tmp/tmp.8UzsnYi7xn /tmp/tmp.3S8gzEtUF7
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZIKtSxrj4i
++ mktemp
+ local LAST_ERR=/tmp/tmp.uG5lmS6NFk
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZIKtSxrj4i
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("685f7c31-695d-4d85-aae7-5824064d78d6") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.uG5lmS6NFk
+ rm /tmp/tmp.ZIKtSxrj4i /tmp/tmp.uG5lmS6NFk
+ return 0
+ run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test3.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ab56ImIkzi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RvBNPybkSE
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ab56ImIkzi
++ cat /tmp/tmp.RvBNPybkSE
++ rm /tmp/tmp.ab56ImIkzi /tmp/tmp.RvBNPybkSE
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7MtIyQxgL6
++ mktemp
+ local LAST_ERR=/tmp/tmp.NvpSfQvAn6
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7MtIyQxgL6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("943cafb2-9ff4-42d9-89d0-c892ce6c9143") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NvpSfQvAn6
+ rm /tmp/tmp.7MtIyQxgL6 /tmp/tmp.NvpSfQvAn6
+ return 0
+ custom_user_name=test1user
+ custom_role_name=test1role
+ run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lm9dFfgXxk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5SsYETIsVS
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lm9dFfgXxk
++ cat /tmp/tmp.5SsYETIsVS
++ rm /tmp/tmp.Lm9dFfgXxk /tmp/tmp.5SsYETIsVS
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.j33f2RgIn4
++ mktemp
+ local LAST_ERR=/tmp/tmp.ql0jk20NGC
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j33f2RgIn4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9f728732-bedb-460f-83e5-5068ac13c30a") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
Successfully added user: { "user" : "test1user", "roles" : [ ] }
bye
+ cat /tmp/tmp.ql0jk20NGC
+ rm /tmp/tmp.j33f2RgIn4 /tmp/tmp.ql0jk20NGC
+ return 0
+ run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hTYlbTslUy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x5TPIz6QzX
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hTYlbTslUy
++ cat /tmp/tmp.x5TPIz6QzX
++ rm /tmp/tmp.hTYlbTslUy /tmp/tmp.x5TPIz6QzX
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aj5pKDV18c
++ mktemp
+ local LAST_ERR=/tmp/tmp.m0KCn3AsjY
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aj5pKDV18c
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("fc3f287e-9a69-4f17-b8f3-51b1f30ba459") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
{ "role" : "test1role", "privileges" : [ ], "roles" : [ ] }
bye
+ cat /tmp/tmp.m0KCn3AsjY
+ rm /tmp/tmp.aj5pKDV18c /tmp/tmp.m0KCn3AsjY
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:32:16+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lSzPG8x0Eu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oHgA01YOSQ
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lSzPG8x0Eu
++ cat /tmp/tmp.oHgA01YOSQ
++ rm /tmp/tmp.lSzPG8x0Eu /tmp/tmp.oHgA01YOSQ
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UT9A4ue0FS
++ mktemp
+ local LAST_ERR=/tmp/tmp.BePI8JWHEX
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UT9A4ue0FS
+ cat /tmp/tmp.BePI8JWHEX
+ rm /tmp/tmp.UT9A4ue0FS /tmp/tmp.BePI8JWHEX
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:32:19+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dnpHn5wXgC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2mlmT1Mv96
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dnpHn5wXgC
++ cat /tmp/tmp.2mlmT1Mv96
++ rm /tmp/tmp.dnpHn5wXgC /tmp/tmp.2mlmT1Mv96
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EhokA9IlBs
++ mktemp
+ local LAST_ERR=/tmp/tmp.MJARqq26y3
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EhokA9IlBs
+ cat /tmp/tmp.MJARqq26y3
+ rm /tmp/tmp.EhokA9IlBs /tmp/tmp.MJARqq26y3
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:32:22+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SCFwK2uTy2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xW8Si6axV6
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SCFwK2uTy2
++ cat /tmp/tmp.xW8Si6axV6
++ rm /tmp/tmp.SCFwK2uTy2 /tmp/tmp.xW8Si6axV6
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nwfV18Gzlk
++ mktemp
+ local LAST_ERR=/tmp/tmp.KaDa5s4T9M
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nwfV18Gzlk
+ cat /tmp/tmp.KaDa5s4T9M
+ rm /tmp/tmp.nwfV18Gzlk /tmp/tmp.KaDa5s4T9M
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2026-07-22T13:31:16.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2026-07-22T13:30:44.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2026-07-22T13:31:47.000+0000 I listening for the commands

+ '[' -z '' ']'
+ desc 'Check GCS profiles'
+ set +o xtrace


-----------------------------------------------------------------------------------
Check GCS profiles
-----------------------------------------------------------------------------------

+ compare_pbm_profile_setup some-name gcp-cs-s3
+ local cluster=some-name
+ local profile=gcp-cs-s3
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == mongod ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
++ mktemp
+ local LAST_OUT=/tmp/tmp.3qSj2eXPj0
++ mktemp
+ local LAST_ERR=/tmp/tmp.8lixAvVBdN
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-s3')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3qSj2eXPj0
+ cat /tmp/tmp.8lixAvVBdN
+ rm /tmp/tmp.3qSj2eXPj0 /tmp/tmp.8lixAvVBdN
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.QVjC8nwngm/pbm_profile_gcp-cs-s3.yml
+ compare_pbm_profile_setup some-name gcp-cs-sa
+ local cluster=some-name
+ local profile=gcp-cs-sa
+ local container=backup-agent
+ local pbm_binary=pbm
+ [[ backup-agent == mongod ]]
+ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
++ mktemp
+ local LAST_OUT=/tmp/tmp.q5KLAVdjWz
++ mktemp
+ local LAST_ERR=/tmp/tmp.XWLV1kdf2c
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-sa')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q5KLAVdjWz
+ cat /tmp/tmp.XWLV1kdf2c
+ rm /tmp/tmp.q5KLAVdjWz /tmp/tmp.XWLV1kdf2c
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.QVjC8nwngm/pbm_profile_gcp-cs-sa.yml
+ backup_name_minio=backup-minio
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3
+ backup_name_gcp_s3=backup-gcp-cs-s3
+ backup_name_gcp_sa=backup-gcp-cs-sa
+ backup_name_azure=backup-azure-blob
+ backup_name_oss_native=backup-oss-native
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-07-22T13:32:31+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jiGvYUHKMw
++ mktemp
+ local LAST_ERR=/tmp/tmp.ulZ3C7bop9
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jiGvYUHKMw
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.ulZ3C7bop9
+ rm /tmp/tmp.jiGvYUHKMw /tmp/tmp.ulZ3C7bop9
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-07-22T13:32:34+0000] running backup backup-gcp-cs-s3
+ yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Cd082U8VgE
++ mktemp
+ local LAST_ERR=/tmp/tmp.uxHxXoZY1f
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cd082U8VgE
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.uxHxXoZY1f
+ rm /tmp/tmp.Cd082U8VgE /tmp/tmp.uxHxXoZY1f
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-07-22T13:32:36+0000] running backup backup-gcp-cs-sa
+ yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kWlaObVuNm
++ mktemp
+ local LAST_ERR=/tmp/tmp.XwGimT8nX8
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kWlaObVuNm
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.XwGimT8nX8
+ rm /tmp/tmp.kWlaObVuNm /tmp/tmp.XwGimT8nX8
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-07-22T13:32:38+0000] running backup backup-azure-blob
+ kubectl_bin apply -f -
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-azure-blob.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8v38DdlYYC
++ mktemp
+ local LAST_ERR=/tmp/tmp.KKvrzy7xWG
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8v38DdlYYC
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.KKvrzy7xWG
+ rm /tmp/tmp.8v38DdlYYC /tmp/tmp.KKvrzy7xWG
+ return 0
+ run_backup oss-native
+ local storage=oss-native
+ local backup_name=backup-oss-native
+ local type=logical
+ log 'running backup backup-oss-native'
+ set +o xtrace
[2026-07-22T13:32:40+0000] running backup backup-oss-native
+ yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-oss-native.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KiWsj2MWdo
++ mktemp
+ local LAST_ERR=/tmp/tmp.wQM0IzM1cu
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KiWsj2MWdo
perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created
+ cat /tmp/tmp.wQM0IzM1cu
+ rm /tmp/tmp.KiWsj2MWdo /tmp/tmp.wQM0IzM1cu
+ return 0
+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-07-22T13:32:43+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PyOn5Ee9Hu
++ mktemp
+ local LAST_ERR=/tmp/tmp.Tcqan66G7i
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PyOn5Ee9Hu
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.Tcqan66G7i
+ rm /tmp/tmp.PyOn5Ee9Hu /tmp/tmp.Tcqan66G7i
+ return 0
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state..OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state...........OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state..................OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state......OK
+ wait_backup backup-oss-native
+ local backup_name=backup-oss-native
+ local target_state=ready
+ set +o xtrace
waiting for backup-oss-native to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|oci://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.WThoBFhqgs
++ sed 's|oss://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BjfW5NUb17
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WThoBFhqgs
++ cat /tmp/tmp.BjfW5NUb17
++ rm /tmp/tmp.WThoBFhqgs /tmp/tmp.BjfW5NUb17
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z
+ setup_aws_credentials
+ local secret_name=aws-s3-secret
+ [[ -n AKIARXP3OARBMEQ6FSEW ]]
+ [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]]
+ echo 'AWS credentials already set in environment'
AWS credentials already set in environment
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-07-22T13:32:34Z
+ key=/rs0/myApp.test.gz
+ endpoint=
+ storage_name=aws-s3
+ retry=0
+ endpoint_arg=()
+ local endpoint_arg
+ [[ -n '' ]]
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-07-22T13:32:34Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z//rs0/myApp.test.gz found in aws-s3'
Backup operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z//rs0/myApp.test.gz found in aws-s3
+ run_recovery_check backup-aws-s3 some-name-rs0
+ local backup=backup-aws-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GMG4hj8dvX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z0yoXl4ArV
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GMG4hj8dvX
++ cat /tmp/tmp.Z0yoXl4ArV
++ rm /tmp/tmp.GMG4hj8dvX /tmp/tmp.Z0yoXl4ArV
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mfK2RSUnZk
++ mktemp
+ local LAST_ERR=/tmp/tmp.x0qeE1JZFM
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mfK2RSUnZk
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a3ef0804-8aec-4c0c-a996-82b2aaceb033") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.x0qeE1JZFM
+ rm /tmp/tmp.mfK2RSUnZk /tmp/tmp.x0qeE1JZFM
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:34:07+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.puALqUfx6I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MvFREvnLi2
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.puALqUfx6I
++ cat /tmp/tmp.MvFREvnLi2
++ rm /tmp/tmp.puALqUfx6I /tmp/tmp.MvFREvnLi2
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VFvINhNLM7
++ mktemp
+ local LAST_ERR=/tmp/tmp.FxR0SQmuAf
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VFvINhNLM7
+ cat /tmp/tmp.FxR0SQmuAf
+ rm /tmp/tmp.VFvINhNLM7 /tmp/tmp.FxR0SQmuAf
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-aws-s3'
+ set +o xtrace
[2026-07-22T13:34:10+0000] running restore restore-backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ local LAST_OUT=/tmp/tmp.rgUaHMDIvv
++ mktemp
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ local LAST_ERR=/tmp/tmp.QX6kzxyiPf
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rgUaHMDIvv
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.QX6kzxyiPf
+ rm /tmp/tmp.rgUaHMDIvv /tmp/tmp.QX6kzxyiPf
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mkjP5j08u6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eFLve0BPjv
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mkjP5j08u6
++ cat /tmp/tmp.eFLve0BPjv
++ rm /tmp/tmp.mkjP5j08u6 /tmp/tmp.eFLve0BPjv
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:34:43+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4O2QsEmnpy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FqbiBNFRYD
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4O2QsEmnpy
++ cat /tmp/tmp.FqbiBNFRYD
++ rm /tmp/tmp.4O2QsEmnpy /tmp/tmp.FqbiBNFRYD
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5Cql64zKsV
++ mktemp
+ local LAST_ERR=/tmp/tmp.rMZkKnSSJC
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5Cql64zKsV
+ cat /tmp/tmp.rMZkKnSSJC
+ rm /tmp/tmp.5Cql64zKsV /tmp/tmp.rMZkKnSSJC
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:34:45+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rafj8eUMxO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yMB4bxTHcY
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Rafj8eUMxO
++ cat /tmp/tmp.yMB4bxTHcY
++ rm /tmp/tmp.Rafj8eUMxO /tmp/tmp.yMB4bxTHcY
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qRZ1LVZVAh
++ mktemp
+ local LAST_ERR=/tmp/tmp.R1xHLJ0aqS
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qRZ1LVZVAh
+ cat /tmp/tmp.R1xHLJ0aqS
+ rm /tmp/tmp.qRZ1LVZVAh /tmp/tmp.R1xHLJ0aqS
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:34:48+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cQ6Vsb2DgM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q7jHpXwp1G
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cQ6Vsb2DgM
++ cat /tmp/tmp.q7jHpXwp1G
++ rm /tmp/tmp.cQ6Vsb2DgM /tmp/tmp.q7jHpXwp1G
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CLNkaxHFnQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.HAkHiej0cz
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CLNkaxHFnQ
+ cat /tmp/tmp.HAkHiej0cz
+ rm /tmp/tmp.CLNkaxHFnQ /tmp/tmp.HAkHiej0cz
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'check backup and restore -- gcp-cs-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-s3
++ local backup_name=backup-gcp-cs-s3
++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
+++ mktemp
++ sed -e 's/.json$//'
++ local LAST_OUT=/tmp/tmp.RHgFZethCD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9dkAlQDyLL
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-gcp-cs-s3' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ sed 's|azure://||'
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ sed 's|gs://||'
+++ seq 0 2
++ sed 's|oci://||'
++ sed 's|s3://||'
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}'
++ sed 's|oss://||'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RHgFZethCD
++ cat /tmp/tmp.9dkAlQDyLL
++ rm /tmp/tmp.RHgFZethCD /tmp/tmp.9dkAlQDyLL
++ return 0
+ backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z
+ setup_gcs_credentials
+ local secret_name=gcp-cs-secret
+ gsutil ls
+ echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret'
Setting up GCS credentials from K8s secret: gcp-cs-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'GCS credentials configured successfully'
GCS credentials configured successfully
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-s3 some-name-rs0
+ local backup=backup-gcp-cs-s3
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3o3H770pgW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fEnRZS3QFO
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3o3H770pgW
++ cat /tmp/tmp.fEnRZS3QFO
++ rm /tmp/tmp.3o3H770pgW /tmp/tmp.fEnRZS3QFO
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kr0ksXzDLN
++ mktemp
+ local LAST_ERR=/tmp/tmp.qHGOcSECiA
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kr0ksXzDLN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b18e66be-b71a-4a24-8de8-7475d076613b") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.qHGOcSECiA
+ rm /tmp/tmp.Kr0ksXzDLN /tmp/tmp.qHGOcSECiA
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:34:59+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.syIhKJZmkY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DJHlh11XnD
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.syIhKJZmkY
++ cat /tmp/tmp.DJHlh11XnD
++ rm /tmp/tmp.syIhKJZmkY /tmp/tmp.DJHlh11XnD
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZoEb2OnYAd
++ mktemp
+ local LAST_ERR=/tmp/tmp.LpinVFAIup
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZoEb2OnYAd
+ cat /tmp/tmp.LpinVFAIup
+ rm /tmp/tmp.ZoEb2OnYAd /tmp/tmp.LpinVFAIup
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-s3'
+ set +o xtrace
[2026-07-22T13:35:02+0000] running restore restore-backup-gcp-cs-s3
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ep0RCkEA99
++ mktemp
+ local LAST_ERR=/tmp/tmp.y4nJzHQlm5
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ep0RCkEA99
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created
+ cat /tmp/tmp.y4nJzHQlm5
+ rm /tmp/tmp.Ep0RCkEA99 /tmp/tmp.y4nJzHQlm5
+ return 0
+ wait_restore backup-gcp-cs-s3 some-name
+ local backup_name=backup-gcp-cs-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QK1BNnw7O5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NT1eQqnPeJ
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QK1BNnw7O5
++ cat /tmp/tmp.NT1eQqnPeJ
++ rm /tmp/tmp.QK1BNnw7O5 /tmp/tmp.NT1eQqnPeJ
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:35:29+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ local LAST_OUT=/tmp/tmp.TY5dyVupjf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iZ0KXkJGgL
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TY5dyVupjf
++ cat /tmp/tmp.iZ0KXkJGgL
++ rm /tmp/tmp.TY5dyVupjf /tmp/tmp.iZ0KXkJGgL
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DdRCp0VHez
++ mktemp
+ local LAST_ERR=/tmp/tmp.dRveNQGmJA
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DdRCp0VHez
+ cat /tmp/tmp.dRveNQGmJA
+ rm /tmp/tmp.DdRCp0VHez /tmp/tmp.dRveNQGmJA
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:35:32+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yIBd9rhl5x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YyVuVIpEYW
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yIBd9rhl5x
++ cat /tmp/tmp.YyVuVIpEYW
++ rm /tmp/tmp.yIBd9rhl5x /tmp/tmp.YyVuVIpEYW
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uV1yLLuqCr
++ mktemp
+ local LAST_ERR=/tmp/tmp.LuOwVHJJWw
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uV1yLLuqCr
+ cat /tmp/tmp.LuOwVHJJWw
+ rm /tmp/tmp.uV1yLLuqCr /tmp/tmp.LuOwVHJJWw
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:35:34+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bxs1OACXWV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FqPbGydHUY
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bxs1OACXWV
++ cat /tmp/tmp.FqPbGydHUY
++ rm /tmp/tmp.bxs1OACXWV /tmp/tmp.FqPbGydHUY
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XeHhjYwLGn
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZlyG2TA6JY
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XeHhjYwLGn
+ cat /tmp/tmp.ZlyG2TA6JY
+ rm /tmp/tmp.XeHhjYwLGn /tmp/tmp.ZlyG2TA6JY
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'check backup and restore -- gcp-cs-sa'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs-sa
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs-sa
++ local backup_name=backup-gcp-cs-sa
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.rZOTlD7naq
++ sed 's|oci://||'
+++ mktemp
++ sed 's|oss://||'
++ local LAST_ERR=/tmp/tmp.bOrRz2BpZb
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-gcp-cs-sa' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rZOTlD7naq
++ cat /tmp/tmp.bOrRz2BpZb
++ rm /tmp/tmp.rZOTlD7naq /tmp/tmp.bOrRz2BpZb
++ return 0
+ backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z
+ check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz found in gcp-cs
+ run_recovery_check backup-gcp-cs-sa some-name-rs0
+ local backup=backup-gcp-cs-sa
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aiG1ptM7MP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b9kDkcOkeV
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aiG1ptM7MP
++ cat /tmp/tmp.b9kDkcOkeV
++ rm /tmp/tmp.aiG1ptM7MP /tmp/tmp.b9kDkcOkeV
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wgtD4M80Qd
++ mktemp
+ local LAST_ERR=/tmp/tmp.6csg5938wQ
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wgtD4M80Qd
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("6224d15f-0532-4ac3-b1e3-69a53c336b4e") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.6csg5938wQ
+ rm /tmp/tmp.wgtD4M80Qd /tmp/tmp.6csg5938wQ
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:35:42+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yoPFuq6Js0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KxbIcIAMJD
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yoPFuq6Js0
++ cat /tmp/tmp.KxbIcIAMJD
++ rm /tmp/tmp.yoPFuq6Js0 /tmp/tmp.KxbIcIAMJD
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SftoTDSUr0
++ mktemp
+ local LAST_ERR=/tmp/tmp.p2Ub1ExogF
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SftoTDSUr0
+ cat /tmp/tmp.p2Ub1ExogF
+ rm /tmp/tmp.SftoTDSUr0 /tmp/tmp.p2Ub1ExogF
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-gcp-cs-sa'
+ set +o xtrace
[2026-07-22T13:35:45+0000] running restore restore-backup-gcp-cs-sa
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RSkR6Qjv7x
++ mktemp
+ local LAST_ERR=/tmp/tmp.eYmy3ZNK8I
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RSkR6Qjv7x
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created
+ cat /tmp/tmp.eYmy3ZNK8I
+ rm /tmp/tmp.RSkR6Qjv7x /tmp/tmp.eYmy3ZNK8I
+ return 0
+ wait_restore backup-gcp-cs-sa some-name
+ local backup_name=backup-gcp-cs-sa
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XU1gd0kuB2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dcBwgfLyOn
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XU1gd0kuB2
++ cat /tmp/tmp.dcBwgfLyOn
++ rm /tmp/tmp.XU1gd0kuB2 /tmp/tmp.dcBwgfLyOn
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:36:18+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oXnU3NBRxH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XssLDEbVkf
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oXnU3NBRxH
++ cat /tmp/tmp.XssLDEbVkf
++ rm /tmp/tmp.oXnU3NBRxH /tmp/tmp.XssLDEbVkf
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LcdgvG126F
++ mktemp
+ local LAST_ERR=/tmp/tmp.UbKAnit0dy
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LcdgvG126F
+ cat /tmp/tmp.UbKAnit0dy
+ rm /tmp/tmp.LcdgvG126F /tmp/tmp.UbKAnit0dy
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:36:20+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oKfviqBEgM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aiMdS2yDTB
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oKfviqBEgM
++ cat /tmp/tmp.aiMdS2yDTB
++ rm /tmp/tmp.oKfviqBEgM /tmp/tmp.aiMdS2yDTB
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.r7iDN6K8Ln
++ mktemp
+ local LAST_ERR=/tmp/tmp.rZp6SW0wXs
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r7iDN6K8Ln
+ cat /tmp/tmp.rZp6SW0wXs
+ rm /tmp/tmp.r7iDN6K8Ln /tmp/tmp.rZp6SW0wXs
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:36:22+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8obIZqdqUp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ihPLIIUC7J
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8obIZqdqUp
++ cat /tmp/tmp.ihPLIIUC7J
++ rm /tmp/tmp.8obIZqdqUp /tmp/tmp.ihPLIIUC7J
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BybQiEJ7yn
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ned67DCkK8
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BybQiEJ7yn
+ cat /tmp/tmp.Ned67DCkK8
+ rm /tmp/tmp.BybQiEJ7yn /tmp/tmp.Ned67DCkK8
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ sed 's|oss://||'
++ sed 's|oci://||'
++ local LAST_OUT=/tmp/tmp.nVkEUfMQ6O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xWAS8B5lFI
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nVkEUfMQ6O
++ cat /tmp/tmp.xWAS8B5lFI
++ rm /tmp/tmp.nVkEUfMQ6O /tmp/tmp.xWAS8B5lFI
++ return 0
+ backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
+ setup_azure_credentials
+ local secret_name=azure-secret
+ echo 'Setting up Azure credentials from K8s secret: azure-secret'
Setting up Azure credentials from K8s secret: azure-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ echo 'Azure credentials configured successfully'
Azure credentials configured successfully
+ check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-07-22T13:33:42Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz found in azure-blob'
Backup psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz found in azure-blob
+ run_recovery_check backup-azure-blob some-name-rs0
+ local backup=backup-azure-blob
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yNbKRpf4TV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.guIf3EDQ1s
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yNbKRpf4TV
++ cat /tmp/tmp.guIf3EDQ1s
++ rm /tmp/tmp.yNbKRpf4TV /tmp/tmp.guIf3EDQ1s
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Xk2qet2X2
++ mktemp
+ local LAST_ERR=/tmp/tmp.UVI9Gr9S3q
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9Xk2qet2X2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d572c7b6-b9ec-4ef9-9065-1039c93c32ab") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.UVI9Gr9S3q
+ rm /tmp/tmp.9Xk2qet2X2 /tmp/tmp.UVI9Gr9S3q
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:36:30+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ipmIf2SdHy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6Zp7TARZ1r
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ipmIf2SdHy
++ cat /tmp/tmp.6Zp7TARZ1r
++ rm /tmp/tmp.ipmIf2SdHy /tmp/tmp.6Zp7TARZ1r
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M5rLu4kb15
++ mktemp
+ local LAST_ERR=/tmp/tmp.wfJdKNZp6x
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M5rLu4kb15
+ cat /tmp/tmp.wfJdKNZp6x
+ rm /tmp/tmp.M5rLu4kb15 /tmp/tmp.wfJdKNZp6x
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-azure-blob'
+ set +o xtrace
[2026-07-22T13:36:32+0000] running restore restore-backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.AM4cn8NrXB
++ mktemp
+ local LAST_ERR=/tmp/tmp.FDBLhsGRLw
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AM4cn8NrXB
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.FDBLhsGRLw
+ rm /tmp/tmp.AM4cn8NrXB /tmp/tmp.FDBLhsGRLw
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YSKveCMCa3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.su35eYg7gy
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YSKveCMCa3
++ cat /tmp/tmp.su35eYg7gy
++ rm /tmp/tmp.YSKveCMCa3 /tmp/tmp.su35eYg7gy
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:37:00+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uOVbIQzO7g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EpBbplZPUC
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uOVbIQzO7g
++ cat /tmp/tmp.EpBbplZPUC
++ rm /tmp/tmp.uOVbIQzO7g /tmp/tmp.EpBbplZPUC
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yf513RDDIu
++ mktemp
+ local LAST_ERR=/tmp/tmp.06RzMtyp8R
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yf513RDDIu
+ cat /tmp/tmp.06RzMtyp8R
+ rm /tmp/tmp.yf513RDDIu /tmp/tmp.06RzMtyp8R
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:37:02+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NE0e4uU1y0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pDO8viNziB
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NE0e4uU1y0
++ cat /tmp/tmp.pDO8viNziB
++ rm /tmp/tmp.NE0e4uU1y0 /tmp/tmp.pDO8viNziB
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9qhSHgkPBj
++ mktemp
+ local LAST_ERR=/tmp/tmp.qDRESy8eY9
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9qhSHgkPBj
+ cat /tmp/tmp.qDRESy8eY9
+ rm /tmp/tmp.9qhSHgkPBj /tmp/tmp.qDRESy8eY9
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:37:05+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ruxmrnFghX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EwqWvOOj7z
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ruxmrnFghX
++ cat /tmp/tmp.EwqWvOOj7z
++ rm /tmp/tmp.ruxmrnFghX /tmp/tmp.EwqWvOOj7z
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RgulqdRe4w
++ mktemp
+ local LAST_ERR=/tmp/tmp.SFmR2JbDxr
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RgulqdRe4w
+ cat /tmp/tmp.SFmR2JbDxr
+ rm /tmp/tmp.RgulqdRe4w /tmp/tmp.SFmR2JbDxr
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'check backup and restore -- oss-native'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- oss-native
-----------------------------------------------------------------------------------

++ get_backup_dest backup-oss-native
++ local backup_name=backup-oss-native
++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}'
++ sed 's|azure://||'
++ sed 's|oss://||'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|gs://||'
++ sed 's|s3://||'
++ local LAST_OUT=/tmp/tmp.Jh51Nn98dn
++ sed 's|oci://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m0hhnMZLSV
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jh51Nn98dn
++ cat /tmp/tmp.m0hhnMZLSV
++ rm /tmp/tmp.Jh51Nn98dn /tmp/tmp.m0hhnMZLSV
++ return 0
+ backup_dest_oss_native=operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
++ get_backup_storage_oss_field backup-oss-native endpointUrl
++ local backup_name=backup-oss-native
++ local field=endpointUrl
++ local cluster_name
++ local storage_name
+++ get_backup_cluster_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MQvMd7bWjW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Ml0gTwUk97
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.MQvMd7bWjW
+++ cat /tmp/tmp.Ml0gTwUk97
+++ rm /tmp/tmp.MQvMd7bWjW /tmp/tmp.Ml0gTwUk97
+++ return 0
++ cluster_name=some-name
+++ get_backup_storage_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FADKqJGtaQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BkM0F9KHUV
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FADKqJGtaQ
+++ cat /tmp/tmp.BkM0F9KHUV
+++ rm /tmp/tmp.FADKqJGtaQ /tmp/tmp.BkM0F9KHUV
+++ return 0
++ storage_name=oss-native
++ kubectl_bin get psmdb some-name -o json
++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PsMd3PtcKX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nAhVOjSOKf
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'json')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PsMd3PtcKX
++ cat /tmp/tmp.nAhVOjSOKf
++ rm /tmp/tmp.PsMd3PtcKX /tmp/tmp.nAhVOjSOKf
++ return 0
+ backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com
++ get_backup_storage_oss_field backup-oss-native region
++ local backup_name=backup-oss-native
++ local field=region
++ local cluster_name
++ local storage_name
+++ get_backup_cluster_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QrUwBrHtXV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZnPpbIznL2
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QrUwBrHtXV
+++ cat /tmp/tmp.ZnPpbIznL2
+++ rm /tmp/tmp.QrUwBrHtXV /tmp/tmp.ZnPpbIznL2
+++ return 0
++ cluster_name=some-name
+++ get_backup_storage_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WiKxJrvYUJ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Y7sLEkMpM8
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WiKxJrvYUJ
+++ cat /tmp/tmp.Y7sLEkMpM8
+++ rm /tmp/tmp.WiKxJrvYUJ /tmp/tmp.Y7sLEkMpM8
+++ return 0
++ storage_name=oss-native
++ kubectl_bin get psmdb some-name -o json
++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.grWpzcnfAr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l1xt7ObI0J
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'json')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.grWpzcnfAr
++ cat /tmp/tmp.l1xt7ObI0J
++ rm /tmp/tmp.grWpzcnfAr /tmp/tmp.l1xt7ObI0J
++ return 0
+ backup_region_oss_native=eu-central-1
+ check_backup_in_storage backup-oss-native oss rs0 myApp.test.gz
+ local backup=backup-oss-native
+ local storage_type=oss
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-oss-native
++ local backup_name=backup-oss-native
++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}'
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|gs://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|oci://||'
++ sed 's|oss://||'
++ local LAST_OUT=/tmp/tmp.h8ncPwTOP0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u3CGTrXYX4
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h8ncPwTOP0
++ cat /tmp/tmp.u3CGTrXYX4
++ rm /tmp/tmp.h8ncPwTOP0 /tmp/tmp.u3CGTrXYX4
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
+ case ${storage_type} in
++ get_backup_storage_oss_field backup-oss-native endpointUrl
++ local backup_name=backup-oss-native
++ local field=endpointUrl
++ local cluster_name
++ local storage_name
+++ get_backup_cluster_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qBDIRVPuzW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FduySnwVcW
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qBDIRVPuzW
+++ cat /tmp/tmp.FduySnwVcW
+++ rm /tmp/tmp.qBDIRVPuzW /tmp/tmp.FduySnwVcW
+++ return 0
++ cluster_name=some-name
+++ get_backup_storage_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JsjwvX7DxV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iyvvedmflG
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.JsjwvX7DxV
+++ cat /tmp/tmp.iyvvedmflG
+++ rm /tmp/tmp.JsjwvX7DxV /tmp/tmp.iyvvedmflG
+++ return 0
++ storage_name=oss-native
++ kubectl_bin get psmdb some-name -o json
++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LUjYxVo3ge
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZ52tyfojn
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'json')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LUjYxVo3ge
++ cat /tmp/tmp.PZ52tyfojn
++ rm /tmp/tmp.LUjYxVo3ge /tmp/tmp.PZ52tyfojn
++ return 0
+ endpoint=https://oss-eu-central-1.aliyuncs.com
++ get_backup_storage_oss_field backup-oss-native region
++ local backup_name=backup-oss-native
++ local field=region
++ local cluster_name
++ local storage_name
+++ get_backup_cluster_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QVbRSndKS6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3LuXBEzuxW
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QVbRSndKS6
+++ cat /tmp/tmp.3LuXBEzuxW
+++ rm /tmp/tmp.QVbRSndKS6 /tmp/tmp.3LuXBEzuxW
+++ return 0
++ cluster_name=some-name
+++ get_backup_storage_name backup-oss-native
+++ local backup_name=backup-oss-native
+++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FBCub9CLew
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iMQlaJG9qQ
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FBCub9CLew
+++ cat /tmp/tmp.iMQlaJG9qQ
+++ rm /tmp/tmp.FBCub9CLew /tmp/tmp.iMQlaJG9qQ
+++ return 0
++ storage_name=oss-native
++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]'
++ kubectl_bin get psmdb some-name -o json
+++ mktemp
++ local LAST_OUT=/tmp/tmp.msu9QH26er
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I8n66aet4P
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'json')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.msu9QH26er
++ cat /tmp/tmp.I8n66aet4P
++ rm /tmp/tmp.msu9QH26er /tmp/tmp.I8n66aet4P
++ return 0
+ region=eu-central-1
++ echo https://oss-eu-central-1.aliyuncs.com
++ /usr/sbin/sed 's|://oss-|://s3.oss-|'
+ endpoint=https://s3.oss-eu-central-1.aliyuncs.com
+ local _orig_key_id=AKIARXP3OARBMEQ6FSEW _orig_secret=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 _orig_default_region= _orig_aws_region=
+ setup_alibaba_cloud_s3_credentials eu-central-1
+ local region=eu-central-1
+ local secret_name=alibaba-cloud-secret
+ echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret'
Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ return 0
+ check_backup_existence_aws operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z /rs0/myApp.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native
++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
+ key=/rs0/myApp.test.gz
+ endpoint=https://s3.oss-eu-central-1.aliyuncs.com
+ storage_name=oss-native
+ retry=0
+ endpoint_arg=()
+ local endpoint_arg
+ [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]]
+ endpoint_arg=(--endpoint-url "$endpoint")
+ aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z/rs0/myApp.test.gz
+ echo 'Backup operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z//rs0/myApp.test.gz found in oss-native'
Backup operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z//rs0/myApp.test.gz found in oss-native
+ AWS_ACCESS_KEY_ID=AKIARXP3OARBMEQ6FSEW
+ AWS_SECRET_ACCESS_KEY=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
+ AWS_DEFAULT_REGION=
+ AWS_REGION=
+ export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION
+ run_recovery_check backup-oss-native some-name-rs0
+ local backup=backup-oss-native
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nCKOvdeN4C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AFtyXAQEEd
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nCKOvdeN4C
++ cat /tmp/tmp.AFtyXAQEEd
++ rm /tmp/tmp.nCKOvdeN4C /tmp/tmp.AFtyXAQEEd
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5UshAUg0d7
++ mktemp
+ local LAST_ERR=/tmp/tmp.vRjfmntxMG
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5UshAUg0d7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("31cd844e-9a78-4f5c-bbb7-c15ccbc6ddf2") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vRjfmntxMG
+ rm /tmp/tmp.5UshAUg0d7 /tmp/tmp.vRjfmntxMG
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:37:28+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kb83Xc5aN0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kUESo1ehfW
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kb83Xc5aN0
++ cat /tmp/tmp.kUESo1ehfW
++ rm /tmp/tmp.kb83Xc5aN0 /tmp/tmp.kUESo1ehfW
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.KKlpq8dI6x
++ mktemp
+ local LAST_ERR=/tmp/tmp.5P8xk9n4yh
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KKlpq8dI6x
+ cat /tmp/tmp.5P8xk9n4yh
+ rm /tmp/tmp.KKlpq8dI6x /tmp/tmp.5P8xk9n4yh
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-oss-native
+ local backup_name=backup-oss-native
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-oss-native'
+ set +o xtrace
[2026-07-22T13:37:31+0000] running restore restore-backup-oss-native
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-oss-native/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-oss-native/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xod5P5yUOq
++ mktemp
+ local LAST_ERR=/tmp/tmp.yWMDRDrDDI
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xod5P5yUOq
perconaservermongodbrestore.psmdb.percona.com/restore-backup-oss-native created
+ cat /tmp/tmp.yWMDRDrDDI
+ rm /tmp/tmp.xod5P5yUOq /tmp/tmp.yWMDRDrDDI
+ return 0
+ wait_restore backup-oss-native some-name
+ local backup_name=backup-oss-native
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-oss-native object to be created.OK
Waiting psmdb-restore/restore-backup-oss-native to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jQGtSDzXvI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3jDkTc2ooF
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jQGtSDzXvI
++ cat /tmp/tmp.3jDkTc2ooF
++ rm /tmp/tmp.jQGtSDzXvI /tmp/tmp.3jDkTc2ooF
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:05+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IbTr0Xxd23
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2bCMdqnj8i
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IbTr0Xxd23
++ cat /tmp/tmp.2bCMdqnj8i
++ rm /tmp/tmp.IbTr0Xxd23 /tmp/tmp.2bCMdqnj8i
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1ZJsIXAWtU
++ mktemp
+ local LAST_ERR=/tmp/tmp.DZlIXRbnzf
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1ZJsIXAWtU
+ cat /tmp/tmp.DZlIXRbnzf
+ rm /tmp/tmp.1ZJsIXAWtU /tmp/tmp.DZlIXRbnzf
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:08+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OgLYd0qe0v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RoqFRE6uKq
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OgLYd0qe0v
++ cat /tmp/tmp.RoqFRE6uKq
++ rm /tmp/tmp.OgLYd0qe0v /tmp/tmp.RoqFRE6uKq
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.VlTV0Oznb9
++ mktemp
+ local LAST_ERR=/tmp/tmp.YhMhyU2EY5
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VlTV0Oznb9
+ cat /tmp/tmp.YhMhyU2EY5
+ rm /tmp/tmp.VlTV0Oznb9 /tmp/tmp.YhMhyU2EY5
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:10+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LMNgKwPAiZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tlOXZVch7L
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LMNgKwPAiZ
++ cat /tmp/tmp.tlOXZVch7L
++ rm /tmp/tmp.LMNgKwPAiZ /tmp/tmp.tlOXZVch7L
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LIFv1d60UP
++ mktemp
+ local LAST_ERR=/tmp/tmp.X5omeK5DDG
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LIFv1d60UP
+ cat /tmp/tmp.X5omeK5DDG
+ rm /tmp/tmp.LIFv1d60UP /tmp/tmp.X5omeK5DDG
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local protocol=http
+ local endpoint
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed -e 's/.json$//'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ sed 's|oci://||'
++ sed 's|oss://||'
++ local LAST_OUT=/tmp/tmp.vbmzsLavdO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bSARAX22d3
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vbmzsLavdO
++ cat /tmp/tmp.bSARAX22d3
++ rm /tmp/tmp.vbmzsLavdO /tmp/tmp.bSARAX22d3
++ return 0
+ backup_dest=operator-testing/2026-07-22T13:32:45Z
+ case ${storage_type} in
+ endpoint=minio-service
+ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/2026-07-22T13:32:45Z/rs0/myApp.test.gz
+ local endpoint=http://minio-service:9000
+ shift
+ grep myApp.test.gz
+ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-07-22T13:32:45Z/rs0/myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ol51JvhQ3H
++ mktemp
+ local LAST_ERR=/tmp/tmp.71pYm46Ogy
+ local exit_status=0
+ local timeout=4
+ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/2026-07-22T13:32:45Z/rs0/myApp.test.gz')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-07-22T13:32:45Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ol51JvhQ3H
+ cat /tmp/tmp.71pYm46Ogy
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-7490
+ rm /tmp/tmp.Ol51JvhQ3H /tmp/tmp.71pYm46Ogy
+ return 0
2026-07-22 13:32:49         56 myApp.test.gz
+ run_recovery_check backup-minio some-name-rs0
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U5cbzB6BcE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7LaSADdokd
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U5cbzB6BcE
++ cat /tmp/tmp.7LaSADdokd
++ rm /tmp/tmp.U5cbzB6BcE /tmp/tmp.7LaSADdokd
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.e2vn8rjbUR
++ mktemp
+ local LAST_ERR=/tmp/tmp.FYkyLraNaU
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.e2vn8rjbUR
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("f97e8f6e-8f16-4e43-8138-d2d335550e36") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.FYkyLraNaU
+ rm /tmp/tmp.e2vn8rjbUR /tmp/tmp.FYkyLraNaU
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:21+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p64qOS8U5w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WFZYA5ZTAe
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p64qOS8U5w
++ cat /tmp/tmp.WFZYA5ZTAe
++ rm /tmp/tmp.p64qOS8U5w /tmp/tmp.WFZYA5ZTAe
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kvRHYcm3LJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.NDiTrbCFrh
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kvRHYcm3LJ
+ cat /tmp/tmp.NDiTrbCFrh
+ rm /tmp/tmp.kvRHYcm3LJ /tmp/tmp.NDiTrbCFrh
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore backup-minio
+ local backup_name=backup-minio
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ log 'running restore restore-backup-minio'
+ set +o xtrace
[2026-07-22T13:38:23+0000] running restore restore-backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.niMSwznNNO
++ mktemp
+ local LAST_ERR=/tmp/tmp.0SoGAwfTQJ
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.niMSwznNNO
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.0SoGAwfTQJ
+ rm /tmp/tmp.niMSwznNNO /tmp/tmp.0SoGAwfTQJ
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nh8nKgCZxj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4eYmBgA3lY
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nh8nKgCZxj
++ cat /tmp/tmp.4eYmBgA3lY
++ rm /tmp/tmp.Nh8nKgCZxj /tmp/tmp.4eYmBgA3lY
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:49+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Uw8w4fWJbj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f8PTbb84BM
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Uw8w4fWJbj
++ cat /tmp/tmp.f8PTbb84BM
++ rm /tmp/tmp.Uw8w4fWJbj /tmp/tmp.f8PTbb84BM
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.98N4BbqpPi
++ mktemp
+ local LAST_ERR=/tmp/tmp.eiB92mtnP0
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.98N4BbqpPi
+ cat /tmp/tmp.eiB92mtnP0
+ rm /tmp/tmp.98N4BbqpPi /tmp/tmp.eiB92mtnP0
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:51+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ulGYrnewRx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FCnLdlHMH4
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ulGYrnewRx
++ cat /tmp/tmp.FCnLdlHMH4
++ rm /tmp/tmp.ulGYrnewRx /tmp/tmp.FCnLdlHMH4
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.B9t38heOP8
++ mktemp
+ local LAST_ERR=/tmp/tmp.I2rTUE5O6a
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B9t38heOP8
+ cat /tmp/tmp.I2rTUE5O6a
+ rm /tmp/tmp.B9t38heOP8 /tmp/tmp.I2rTUE5O6a
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:38:53+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZY2wcHClTV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9kwT6TjOxC
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZY2wcHClTV
++ cat /tmp/tmp.9kwT6TjOxC
++ rm /tmp/tmp.ZY2wcHClTV /tmp/tmp.9kwT6TjOxC
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ks6fvSOteH
++ mktemp
+ local LAST_ERR=/tmp/tmp.p5lxVegwC1
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ks6fvSOteH
+ cat /tmp/tmp.p5lxVegwC1
+ rm /tmp/tmp.ks6fvSOteH /tmp/tmp.p5lxVegwC1
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.dropUser("test1user")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q4NJkbXSjo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nGobOUykeF
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q4NJkbXSjo
++ cat /tmp/tmp.nGobOUykeF
++ rm /tmp/tmp.Q4NJkbXSjo /tmp/tmp.nGobOUykeF
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.spp9KSBinO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xdry7DyGoH
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.spp9KSBinO
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9765eb15-b179-4b86-8c8a-21cb0d558816") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.Xdry7DyGoH
+ rm /tmp/tmp.spp9KSBinO /tmp/tmp.Xdry7DyGoH
+ return 0
+ run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.dropRole("test1role")'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.87IGh3o2By
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zHRLS1zqVD
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.87IGh3o2By
++ cat /tmp/tmp.zHRLS1zqVD
++ rm /tmp/tmp.87IGh3o2By /tmp/tmp.zHRLS1zqVD
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rGVIZk9j7R
++ mktemp
+ local LAST_ERR=/tmp/tmp.r4thQs6wlw
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rGVIZk9j7R
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d3620ca5-7fe5-4b08-91df-1715f5642260") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.r4thQs6wlw
+ rm /tmp/tmp.rGVIZk9j7R /tmp/tmp.r4thQs6wlw
+ return 0
+ desc 'selective restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=false
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g6OLjVRrP0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ufu5Q2Im9d
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g6OLjVRrP0
++ cat /tmp/tmp.Ufu5Q2Im9d
++ rm /tmp/tmp.g6OLjVRrP0 /tmp/tmp.Ufu5Q2Im9d
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.loAENZP9Yt
++ mktemp
+ local LAST_ERR=/tmp/tmp.AcNWbaLwqW
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.loAENZP9Yt
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7f2628df-f9b2-436f-8c42-5d169055f1a1") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.AcNWbaLwqW
+ rm /tmp/tmp.loAENZP9Yt /tmp/tmp.AcNWbaLwqW
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ jq 'index("test2") != null'
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SkbUOdullw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kRcJRL1scE
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SkbUOdullw
+++ cat /tmp/tmp.kRcJRL1scE
+++ rm /tmp/tmp.SkbUOdullw /tmp/tmp.kRcJRL1scE
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t2BTezrI9m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Xk2NGbper
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t2BTezrI9m
++ cat /tmp/tmp.1Xk2NGbper
++ rm /tmp/tmp.t2BTezrI9m /tmp/tmp.1Xk2NGbper
++ return 0
+ [[ false == true ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=false
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VFsIdUZUfC
++ mktemp
+ local LAST_ERR=/tmp/tmp.FVD2xsF3Yc
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VFsIdUZUfC
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.FVD2xsF3Yc
+ rm /tmp/tmp.VFsIdUZUfC /tmp/tmp.FVD2xsF3Yc
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W831pyXeGp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WI0RuNnG0T
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W831pyXeGp
++ cat /tmp/tmp.WI0RuNnG0T
++ rm /tmp/tmp.W831pyXeGp /tmp/tmp.WI0RuNnG0T
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dUGL1tkSG3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cxamdJZ604
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dUGL1tkSG3
+++ cat /tmp/tmp.cxamdJZ604
+++ rm /tmp/tmp.dUGL1tkSG3 /tmp/tmp.cxamdJZ604
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gy35NGrTF9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ogvahA894Z
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Gy35NGrTF9
++ cat /tmp/tmp.ogvahA894Z
++ rm /tmp/tmp.Gy35NGrTF9 /tmp/tmp.ogvahA894Z
++ return 0
+ [[ false == true ]]
+ [[ false == true ]]
+ [[ false == false ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
++ jq 'any(.[]; ._id==myApp.test1user)'
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ local LAST_OUT=/tmp/tmp.fCcI1fv8RI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Eri7pe07qd
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fCcI1fv8RI
+++ cat /tmp/tmp.Eri7pe07qd
+++ rm /tmp/tmp.fCcI1fv8RI /tmp/tmp.Eri7pe07qd
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MC7SJX5AUN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.utQrv9eZXs
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MC7SJX5AUN
++ cat /tmp/tmp.utQrv9eZXs
++ rm /tmp/tmp.MC7SJX5AUN /tmp/tmp.utQrv9eZXs
++ return 0
grep: write error: Broken pipe
+ [[ '' == true ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ grep -v 'switched to'
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ jq 'any(.[]; ._id==myApp.test1role)'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kpi6duwbTB
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ local LAST_ERR=/tmp/tmp.z4Ce3worOj
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kpi6duwbTB
+++ cat /tmp/tmp.z4Ce3worOj
+++ rm /tmp/tmp.kpi6duwbTB /tmp/tmp.z4Ce3worOj
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eQmSkLyUDw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5d8IT6xZwG
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eQmSkLyUDw
++ cat /tmp/tmp.5d8IT6xZwG
++ rm /tmp/tmp.eQmSkLyUDw /tmp/tmp.5d8IT6xZwG
++ return 0
grep: write error: Broken pipe
+ [[ '' == true ]]
+ kubectl_bin delete psmdb-restore restore-backup-minio-selective
++ mktemp
+ local LAST_OUT=/tmp/tmp.DKexBIU1Hq
++ mktemp
+ local LAST_ERR=/tmp/tmp.GGaoXZ0FCK
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'psmdb-restore' 'restore-backup-minio-selective')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-restore restore-backup-minio-selective
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DKexBIU1Hq
perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-7490 namespace
+ cat /tmp/tmp.GGaoXZ0FCK
+ rm /tmp/tmp.DKexBIU1Hq /tmp/tmp.GGaoXZ0FCK
+ return 0
+ run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test2.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pTtEpmr2Lv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f4DrMW7XDf
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pTtEpmr2Lv
++ cat /tmp/tmp.f4DrMW7XDf
++ rm /tmp/tmp.pTtEpmr2Lv /tmp/tmp.f4DrMW7XDf
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AjPYw5on7r
++ mktemp
+ local LAST_ERR=/tmp/tmp.lUMwpVMZbn
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AjPYw5on7r
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("0252ffda-748e-4819-b641-907a14d0d139") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.lUMwpVMZbn
+ rm /tmp/tmp.AjPYw5on7r /tmp/tmp.lUMwpVMZbn
+ return 0
+ desc 'selective restore with users and roles -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with users and roles -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local collection=test2
+ local with_users_and_roles=true
+ local username=test1user
+ local role=test1role
+ restore_name=restore-backup-minio-selective
+ run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test2.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kKhtjI7fVF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5jqLMdU7gV
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kKhtjI7fVF
++ cat /tmp/tmp.5jqLMdU7gV
++ rm /tmp/tmp.kKhtjI7fVF /tmp/tmp.5jqLMdU7gV
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.v4lOSXkvvS
++ mktemp
+ local LAST_ERR=/tmp/tmp.GfW9r8nZnG
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v4lOSXkvvS
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("614b84d3-7570-4b2e-8bf6-208366b74ffc") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.GfW9r8nZnG
+ rm /tmp/tmp.v4lOSXkvvS /tmp/tmp.GfW9r8nZnG
+ return 0
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ grep -v 'switched to'
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fBEtZ3sW7m
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.23qoOzjfSN
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fBEtZ3sW7m
+++ cat /tmp/tmp.23qoOzjfSN
+++ rm /tmp/tmp.fBEtZ3sW7m /tmp/tmp.23qoOzjfSN
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0WNqK5qNgu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iOlJdeK0KF
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0WNqK5qNgu
++ cat /tmp/tmp.iOlJdeK0KF
++ rm /tmp/tmp.0WNqK5qNgu /tmp/tmp.iOlJdeK0KF
++ return 0
+ [[ false == true ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-selective"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.namespaces[0]="myApp.test"'
+ yq .spec.selective.withUsersAndRoles=true
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ms3GfdWFB3
++ mktemp
+ local LAST_ERR=/tmp/tmp.goWeWhOBeV
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ms3GfdWFB3
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created
+ cat /tmp/tmp.goWeWhOBeV
+ rm /tmp/tmp.Ms3GfdWFB3 /tmp/tmp.goWeWhOBeV
+ return 0
+ wait_restore backup-minio-selective some-name
+ local backup_name=backup-minio-selective
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK
Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N9R36F9gkw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g5Q93Cn2CY
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N9R36F9gkw
++ cat /tmp/tmp.g5Q93Cn2CY
++ rm /tmp/tmp.N9R36F9gkw /tmp/tmp.g5Q93Cn2CY
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
++ collection_exists test2
++ local collection=test2
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ jq 'index("test2") != null'
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KbmdOKOQgs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xIo2vz6EVM
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.KbmdOKOQgs
+++ cat /tmp/tmp.xIo2vz6EVM
+++ rm /tmp/tmp.KbmdOKOQgs /tmp/tmp.xIo2vz6EVM
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZMs4PFk8Wi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QYsQqe7YCo
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZMs4PFk8Wi
++ cat /tmp/tmp.QYsQqe7YCo
++ rm /tmp/tmp.ZMs4PFk8Wi /tmp/tmp.QYsQqe7YCo
++ return 0
+ [[ false == true ]]
+ [[ true == true ]]
++ user_exists test1user
++ local username=test1user
++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getUsers())'
++ jq 'any(.[]; ._id==myApp.test1user)'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hfv7Ioj49a
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1user)               
jq: 1 compile error
+++ local LAST_ERR=/tmp/tmp.ijlWPMNfIQ
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hfv7Ioj49a
+++ cat /tmp/tmp.ijlWPMNfIQ
+++ rm /tmp/tmp.hfv7Ioj49a /tmp/tmp.ijlWPMNfIQ
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u9KHNCzLX0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DaylHbNbzK
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u9KHNCzLX0
++ cat /tmp/tmp.DaylHbNbzK
++ rm /tmp/tmp.u9KHNCzLX0 /tmp/tmp.DaylHbNbzK
++ return 0
grep: write error: Broken pipe
+ [[ '' == false ]]
++ role_exists test1role
++ local role=test1role
++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ jq 'any(.[]; ._id==myApp.test1role)'
++ local 'command=use myApp\n JSON.stringify(db.getRoles())'
++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.daRzhSwHrL
++++ mktemp
jq: error: myApp/0 is not defined at <top-level>, line 1:
any(.[]; ._id==myApp.test1role)               
jq: 1 compile error
+++ local LAST_ERR=/tmp/tmp.J0sW6gdOI1
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.daRzhSwHrL
+++ cat /tmp/tmp.J0sW6gdOI1
+++ rm /tmp/tmp.daRzhSwHrL /tmp/tmp.J0sW6gdOI1
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZpkSt6FUqq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4fqHef9PZJ
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZpkSt6FUqq
++ cat /tmp/tmp.4fqHef9PZJ
++ rm /tmp/tmp.ZpkSt6FUqq /tmp/tmp.4fqHef9PZJ
++ return 0
grep: write error: Broken pipe
+ [[ '' == false ]]
+ desc 'selective restore with namespace remap -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
selective restore with namespace remap -- minio
-----------------------------------------------------------------------------------

+ run_recovery_check_namespace_remap backup-minio some-name-rs0 myApp test test_remapped
+ local backup=backup-minio
+ local cluster=some-name-rs0
+ local database=myApp
+ local src_collection=test
+ local dst_collection=test_remapped
+ restore_name=restore-backup-minio-remapped
+ run_mongo 'use myApp\n db.test_remapped.drop()' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test_remapped.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JaA8hvtnul
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SYqQqjjDAd
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JaA8hvtnul
++ cat /tmp/tmp.SYqQqjjDAd
++ rm /tmp/tmp.JaA8hvtnul /tmp/tmp.SYqQqjjDAd
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aMrWJt8KnN
++ mktemp
+ local LAST_ERR=/tmp/tmp.MyHHQsPWKM
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aMrWJt8KnN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ae0dc161-0dec-4fb2-9051-609ed4ce1d6d") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.MyHHQsPWKM
+ rm /tmp/tmp.aMrWJt8KnN /tmp/tmp.MyHHQsPWKM
+ return 0
++ collection_exists test_remapped
++ local collection=test_remapped
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ jq 'index("test_remapped") != null'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dVvCzzh93d
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.93eb2vBsEU
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dVvCzzh93d
+++ cat /tmp/tmp.93eb2vBsEU
+++ rm /tmp/tmp.dVvCzzh93d /tmp/tmp.93eb2vBsEU
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a5ltAQoaC4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wnd6jxLD8q
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a5ltAQoaC4
++ cat /tmp/tmp.wnd6jxLD8q
++ rm /tmp/tmp.a5ltAQoaC4 /tmp/tmp.wnd6jxLD8q
++ return 0
+ [[ false == true ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore.yml
+ yq '.metadata.name="restore-backup-minio-remapped"'
+ yq '.spec.backupName="backup-minio"'
+ yq '.spec.selective.nsFrom="myApp.test"'
+ yq '.spec.selective.nsTo="myApp.test_remapped"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RnIZIOFxRR
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Lqf68Y6Yx
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RnIZIOFxRR
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-remapped created
+ cat /tmp/tmp.2Lqf68Y6Yx
+ rm /tmp/tmp.RnIZIOFxRR /tmp/tmp.2Lqf68Y6Yx
+ return 0
+ wait_restore backup-minio-remapped some-name
+ local backup_name=backup-minio-remapped
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-remapped object to be created.OK
Waiting psmdb-restore/restore-backup-minio-remapped to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zMU7WddmBp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HR42RiHPc7
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zMU7WddmBp
++ cat /tmp/tmp.HR42RiHPc7
++ rm /tmp/tmp.zMU7WddmBp /tmp/tmp.HR42RiHPc7
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
++ collection_exists test_remapped
++ local collection=test_remapped
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ grep -v 'switched to'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ jq 'index("test_remapped") != null'
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SpPyoFtTfr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0C0ie2vXoY
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SpPyoFtTfr
+++ cat /tmp/tmp.0C0ie2vXoY
+++ rm /tmp/tmp.SpPyoFtTfr /tmp/tmp.0C0ie2vXoY
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZHsxdiWLWg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YMA5Yqu4fX
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZHsxdiWLWg
++ cat /tmp/tmp.YMA5Yqu4fX
++ rm /tmp/tmp.ZHsxdiWLWg /tmp/tmp.YMA5Yqu4fX
++ return 0
+ [[ true == false ]]
++ collection_exists test
++ local collection=test
++ jq 'index("test") != null'
++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-7490 '' '' --quiet
++ grep -v 'switched to'
++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())'
++ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=--quiet
++ local replica_set=rs0
++ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6jCEOChQ39
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LnP1WV8nuK
+++ local exit_status=0
+++ local timeout=4
+++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
+++ local args
+++ transformed_files=()
+++ local transformed_files
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
+++ arch_scheduling_enabled
+++ [[ -n '' ]]
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.6jCEOChQ39
+++ cat /tmp/tmp.LnP1WV8nuK
+++ rm /tmp/tmp.6jCEOChQ39 /tmp/tmp.LnP1WV8nuK
+++ return 0
++ local client_container=psmdb-client-757cd65695-j5q5m
++ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vAXBofm50U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.94mbhVHinv
++ local exit_status=0
++ local timeout=4
++ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vAXBofm50U
++ cat /tmp/tmp.94mbhVHinv
++ rm /tmp/tmp.vAXBofm50U /tmp/tmp.94mbhVHinv
++ return 0
+ [[ true == false ]]
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -remapped .svc.cluster.local myApp test_remapped
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-remapped
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test_remapped
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test_remapped.find()'
+ [[ -n '' ]]
+ log 'running db.test_remapped.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:02+0000] running db.test_remapped.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test_remapped.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test_remapped.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T8D3NUVtES
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FvhllPmBI0
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T8D3NUVtES
++ cat /tmp/tmp.FvhllPmBI0
++ rm /tmp/tmp.T8D3NUVtES /tmp/tmp.FvhllPmBI0
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TdNO46itm2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y4xZNnOl6Y
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TdNO46itm2
+ cat /tmp/tmp.Y4xZNnOl6Y
+ rm /tmp/tmp.TdNO46itm2 /tmp/tmp.Y4xZNnOl6Y
+ return 0
+ compare_mongo_output find -remapped
+ local command=find
+ local postfix=-remapped
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-remapped.json /tmp/tmp.QVjC8nwngm/find-remapped
+ return
+ desc 'restore from backup source, with storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, with storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.9PYaO0h49n
+++ mktemp
++ sed 's|gs://||'
++ sed 's|oci://||'
++ sed 's|oss://||'
++ local LAST_ERR=/tmp/tmp.t43xirKAsy
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9PYaO0h49n
++ cat /tmp/tmp.t43xirKAsy
++ rm /tmp/tmp.9PYaO0h49n /tmp/tmp.t43xirKAsy
++ return 0
+ backup_dest_minio=operator-testing/2026-07-22T13:32:45Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-07-22T13:32:45Z some-name-rs0 backup-minio-source-0
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-07-22T13:32:45Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-0
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4Ayqo3H0Hu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sZQnMkgDTX
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4Ayqo3H0Hu
++ cat /tmp/tmp.sZQnMkgDTX
++ rm /tmp/tmp.4Ayqo3H0Hu /tmp/tmp.sZQnMkgDTX
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hhjNWyaXP3
++ mktemp
+ local LAST_ERR=/tmp/tmp.wwO6YvUsE9
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hhjNWyaXP3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b077afcb-3736-4a80-afbc-2acfddf59176") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.wwO6YvUsE9
+ rm /tmp/tmp.hhjNWyaXP3 /tmp/tmp.wwO6YvUsE9
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:08+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w3XFJUj1Xu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.THyZQLON7R
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w3XFJUj1Xu
++ cat /tmp/tmp.THyZQLON7R
++ rm /tmp/tmp.w3XFJUj1Xu /tmp/tmp.THyZQLON7R
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aKv4DlQRNW
++ mktemp
+ local LAST_ERR=/tmp/tmp.9F9Qm6XLP2
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aKv4DlQRNW
+ cat /tmp/tmp.9F9Qm6XLP2
+ rm /tmp/tmp.aKv4DlQRNW /tmp/tmp.9F9Qm6XLP2
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore_backupsource backup-minio-source-0 operator-testing/2026-07-22T13:32:45Z
+ local backupName=backup-minio-source-0
+ local backupDest=operator-testing/2026-07-22T13:32:45Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-07-22T13:32:45Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-07-22T13:32:45Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-07-22T13:32:45Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.C1iVUUGL3g
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q0txZFLBoW
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C1iVUUGL3g
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created
+ cat /tmp/tmp.Q0txZFLBoW
+ rm /tmp/tmp.C1iVUUGL3g /tmp/tmp.Q0txZFLBoW
+ return 0
+ return
+ wait_restore backup-minio-source-0 some-name
+ local backup_name=backup-minio-source-0
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FeW7L5EW6v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o56Aj1zq2Y
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FeW7L5EW6v
++ cat /tmp/tmp.o56Aj1zq2Y
++ rm /tmp/tmp.FeW7L5EW6v /tmp/tmp.o56Aj1zq2Y
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:38+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EwwbTGzKOF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ovmZwegInS
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EwwbTGzKOF
++ cat /tmp/tmp.ovmZwegInS
++ rm /tmp/tmp.EwwbTGzKOF /tmp/tmp.ovmZwegInS
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UownK9NDec
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZgnA0UPi5
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UownK9NDec
+ cat /tmp/tmp.MZgnA0UPi5
+ rm /tmp/tmp.UownK9NDec /tmp/tmp.MZgnA0UPi5
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:41+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DXHGb2JYgr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WKmqKXyRkd
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DXHGb2JYgr
++ cat /tmp/tmp.WKmqKXyRkd
++ rm /tmp/tmp.DXHGb2JYgr /tmp/tmp.WKmqKXyRkd
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.55vYEG7DCD
++ mktemp
+ local LAST_ERR=/tmp/tmp.OV9DCHbm64
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.55vYEG7DCD
+ cat /tmp/tmp.OV9DCHbm64
+ rm /tmp/tmp.55vYEG7DCD /tmp/tmp.OV9DCHbm64
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:43+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OHaT3rfmiZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pk6YNUS7PT
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OHaT3rfmiZ
++ cat /tmp/tmp.Pk6YNUS7PT
++ rm /tmp/tmp.OHaT3rfmiZ /tmp/tmp.Pk6YNUS7PT
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.V71K6rhuDt
++ mktemp
+ local LAST_ERR=/tmp/tmp.hQSJltVysd
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V71K6rhuDt
+ cat /tmp/tmp.hQSJltVysd
+ rm /tmp/tmp.V71K6rhuDt /tmp/tmp.hQSJltVysd
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'restore from backup source, no storageName -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
restore from backup source, no storageName -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ sed 's|oss://||'
++ sed 's|oci://||'
++ local LAST_OUT=/tmp/tmp.vspTAUezpF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B1HQSO2C6E
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vspTAUezpF
++ cat /tmp/tmp.B1HQSO2C6E
++ rm /tmp/tmp.vspTAUezpF /tmp/tmp.B1HQSO2C6E
++ return 0
+ backup_dest_minio=operator-testing/2026-07-22T13:32:45Z
+ run_recovery_check_bkp_source backup-minio operator-testing/2026-07-22T13:32:45Z some-name-rs0 backup-minio-source-1
+ local backup=backup-minio
+ local backup_dest=operator-testing/2026-07-22T13:32:45Z
+ local cluster=some-name-rs0
+ local source=backup-minio-source-1
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-7490
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xAQ6bEKg3m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EUPrwqJMvP
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xAQ6bEKg3m
++ cat /tmp/tmp.EUPrwqJMvP
++ rm /tmp/tmp.xAQ6bEKg3m /tmp/tmp.EUPrwqJMvP
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.etYG8AaiJN
++ mktemp
+ local LAST_ERR=/tmp/tmp.v5QFFGKsFu
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.etYG8AaiJN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("b5fae0bd-6b64-48a0-87c7-7ec787b557ae") }
Percona Server for MongoDB server version: v8.0.26-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.v5QFFGKsFu
+ rm /tmp/tmp.etYG8AaiJN /tmp/tmp.v5QFFGKsFu
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-7490 -2nd .svc.cluster.local myApp test
+ local command=find
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:41:49+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-7490 mongodb .svc.cluster.local '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5WB2cUa712
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HONztRAHr3
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5WB2cUa712
++ cat /tmp/tmp.HONztRAHr3
++ rm /tmp/tmp.5WB2cUa712 /tmp/tmp.HONztRAHr3
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ny4urlI11A
++ mktemp
+ local LAST_ERR=/tmp/tmp.dEdJOdrJ7W
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ny4urlI11A
+ cat /tmp/tmp.dEdJOdrJ7W
+ rm /tmp/tmp.ny4urlI11A /tmp/tmp.dEdJOdrJ7W
+ return 0
+ compare_mongo_output find -2nd
+ local command=find
+ local postfix=-2nd
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.QVjC8nwngm/find-2nd
+ return
+ run_restore_backupsource backup-minio-source-1 operator-testing/2026-07-22T13:32:45Z
+ local backupName=backup-minio-source-1
+ local backupDest=operator-testing/2026-07-22T13:32:45Z
+ local storageName=
+ desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-07-22T13:32:45Z'
+ set +o xtrace


-----------------------------------------------------------------------------------
run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-07-22T13:32:45Z
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/restore-backupsource.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/'
+ /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-07-22T13:32:45Z|'
+ /usr/sbin/sed -e /storageName/d
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nQ5pKGqNF9
++ mktemp
+ local LAST_ERR=/tmp/tmp.JkQ6Y4J1nN
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nQ5pKGqNF9
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created
+ cat /tmp/tmp.JkQ6Y4J1nN
+ rm /tmp/tmp.nQ5pKGqNF9 /tmp/tmp.JkQ6Y4J1nN
+ return 0
+ return
+ wait_restore backup-minio-source-1 some-name
+ local backup_name=backup-minio-source-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK
Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KjMzAf85fu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bURT79E2wi
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KjMzAf85fu
++ cat /tmp/tmp.bURT79E2wi
++ rm /tmp/tmp.KjMzAf85fu /tmp/tmp.bURT79E2wi
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ simple_data_check some-name-rs0 3 0
+ local cluster_name=some-name-rs0
+ let last_pod=3-1
+ local isSharded=0
+ local cluster_pfx=
+ '[' 0 -eq 1 ']'
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:42:19+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zx6JtJNnrE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Hy0UfrxVn2
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zx6JtJNnrE
++ cat /tmp/tmp.Hy0UfrxVn2
++ rm /tmp/tmp.zx6JtJNnrE /tmp/tmp.Hy0UfrxVn2
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.L6tewpkQWJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.e6nV7Hs7p3
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L6tewpkQWJ
+ cat /tmp/tmp.e6nV7Hs7p3
+ rm /tmp/tmp.L6tewpkQWJ /tmp/tmp.e6nV7Hs7p3
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:42:22+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HpbXmclDeh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kFaxnlp4Pf
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HpbXmclDeh
++ cat /tmp/tmp.kFaxnlp4Pf
++ rm /tmp/tmp.HpbXmclDeh /tmp/tmp.kFaxnlp4Pf
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tWGoyr1Cxl
++ mktemp
+ local LAST_ERR=/tmp/tmp.3OI23ArU89
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tWGoyr1Cxl
+ cat /tmp/tmp.3OI23ArU89
+ rm /tmp/tmp.tWGoyr1Cxl /tmp/tmp.3OI23ArU89
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ for i in $(seq 0 $last_pod)
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local retries=3
+ local retry_delay=10
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-07-22T13:42:24+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
++ seq 1 3
+ for attempt in $(seq 1 "$retries")
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 mongodb '' '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fbtUYUrdQO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nIFwZfqbCu
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fbtUYUrdQO
++ cat /tmp/tmp.nIFwZfqbCu
++ rm /tmp/tmp.fbtUYUrdQO /tmp/tmp.nIFwZfqbCu
++ return 0
+ local client_container=psmdb-client-757cd65695-j5q5m
+ kubectl_bin exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tIVSw0rLvv
++ mktemp
+ local LAST_ERR=/tmp/tmp.YwWZ40Mu06
+ local exit_status=0
+ local timeout=4
+ args=('exec' 'psmdb-client-757cd65695-j5q5m' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-757cd65695-j5q5m -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tIVSw0rLvv
+ cat /tmp/tmp.YwWZ40Mu06
+ rm /tmp/tmp.tIVSw0rLvv /tmp/tmp.YwWZ40Mu06
+ return 0
+ compare_mongo_output find ''
+ local command=find
+ local postfix=
+ [[ 0 -ne 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/compare/find.json /tmp/tmp.QVjC8nwngm/find
+ return
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.mq1TYzpOeN
++ mktemp
+ local LAST_ERR=/tmp/tmp.ueNmz84Dp0
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'psmdb-backup' '--all')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mq1TYzpOeN
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-7490 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-7490 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-7490 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-7490 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-7490 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-7490 namespace
+ cat /tmp/tmp.ueNmz84Dp0
+ rm /tmp/tmp.mq1TYzpOeN /tmp/tmp.ueNmz84Dp0
+ return 0
++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/
++ local endpoint=http://minio-service:9000
++ shift
++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c operator-testing/2026-07-22T13:32:45Z
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z5WEWcV2jK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bCis93Sc0D
++ local exit_status=0
++ local timeout=4
++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z5WEWcV2jK
++ cat /tmp/tmp.bCis93Sc0D
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
++ rm /tmp/tmp.z5WEWcV2jK /tmp/tmp.bCis93Sc0D
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z /rs0/myApp.test.gz
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:32:34Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup/2026-07-22T13:32:34Z
+ key=/rs0/myApp.test.gz
+ storage_name=aws-s3
+ retry=0
+ aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-07-22T13:32:34Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-07-22T13:32:34Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3'
Backup psmdb-demand-backup/2026-07-22T13:32:34Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:32:57Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z /rs0/myApp.test.gz
+ backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z
+ obj=/rs0/myApp.test.gz
+ storage_name=gcp-cs
+ retry=0
+ gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz
+ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz
+ echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz not found in gcp-cs'
Backup gs://operator-testing/psmdb-demand-backup/2026-07-22T13:33:30Z/rs0/myApp.test.gz not found in gcp-cs
+ check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z /rs0/myApp.test.gz
++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||'
+ url_path=operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ cut -d/ -f1
+ container=operator-testing
++ echo operator-testing/psmdb-demand-backup/2026-07-22T13:33:42Z
++ cut -d/ -f2-
+ blob_prefix=psmdb-demand-backup/2026-07-22T13:33:42Z
+ blob=/rs0/myApp.test.gz
+ storage_name=azure-blob
+ retry=0
+ blob_path=psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz
+ az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob'
Backup psmdb-demand-backup/2026-07-22T13:33:42Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob
+ check_backup_deletion_oss operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1
++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
++ cut -d/ -f1
+ bucket=operator-testing
++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
++ cut -d/ -f2-
+ key_prefix=psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z
+ key=/rs0/myApp.test.gz
++ echo https://oss-eu-central-1.aliyuncs.com
++ /usr/sbin/sed 's|://oss-|://s3.oss-|'
+ endpoint=https://s3.oss-eu-central-1.aliyuncs.com
+ region=eu-central-1
+ storage_name=oss-native
+ retry=0
+ setup_alibaba_cloud_s3_credentials eu-central-1
+ local region=eu-central-1
+ local secret_name=alibaba-cloud-secret
+ echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret'
Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret
+ local trace_was_on=0
+ [[ ehxB == *x* ]]
+ trace_was_on=1
+ set +x
+ return 0
+ aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z/rs0/myApp.test.gz
+ echo 'Backup psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native'
Backup psmdb-demand-backup-oss-native/2026-07-22T13:33:09Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native
+ desc 'checking backup deletion without cr'
+ set +o xtrace


-----------------------------------------------------------------------------------
checking backup deletion without cr
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-07-22T13:42:46+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tAMFdGR6Xo
++ mktemp
+ local LAST_ERR=/tmp/tmp.twywXorRQ0
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tAMFdGR6Xo
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.twywXorRQ0
+ rm /tmp/tmp.tAMFdGR6Xo /tmp/tmp.twywXorRQ0
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ log 'running backup backup-aws-s3'
+ set +o xtrace
[2026-07-22T13:42:49+0000] running backup backup-aws-s3
+ yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8nyqhnPL0l
++ mktemp
+ local LAST_ERR=/tmp/tmp.gnMSNENcA5
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8nyqhnPL0l
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.gnMSNENcA5
+ rm /tmp/tmp.8nyqhnPL0l /tmp/tmp.gnMSNENcA5
+ return 0
+ run_backup gcp-cs-s3
+ local storage=gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local type=logical
+ log 'running backup backup-gcp-cs-s3'
+ set +o xtrace
[2026-07-22T13:42:51+0000] running backup backup-gcp-cs-s3
+ yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.DG3GOksWgw
++ mktemp
+ local LAST_ERR=/tmp/tmp.3mMZAingzB
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DG3GOksWgw
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created
+ cat /tmp/tmp.3mMZAingzB
+ rm /tmp/tmp.DG3GOksWgw /tmp/tmp.3mMZAingzB
+ return 0
+ run_backup gcp-cs-sa
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local type=logical
+ log 'running backup backup-gcp-cs-sa'
+ set +o xtrace
[2026-07-22T13:42:54+0000] running backup backup-gcp-cs-sa
+ yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VaTeInEEF0
++ mktemp
+ local LAST_ERR=/tmp/tmp.el0368gE77
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VaTeInEEF0
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created
+ cat /tmp/tmp.el0368gE77
+ rm /tmp/tmp.VaTeInEEF0 /tmp/tmp.el0368gE77
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ log 'running backup backup-azure-blob'
+ set +o xtrace
[2026-07-22T13:42:55+0000] running backup backup-azure-blob
+ yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ASgoFp8z5R
++ mktemp
+ local LAST_ERR=/tmp/tmp.dmaHeqmYCe
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ASgoFp8z5R
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.dmaHeqmYCe
+ rm /tmp/tmp.ASgoFp8z5R /tmp/tmp.dmaHeqmYCe
+ return 0
+ run_backup oss-native
+ local storage=oss-native
+ local backup_name=backup-oss-native
+ local type=logical
+ log 'running backup backup-oss-native'
+ set +o xtrace
[2026-07-22T13:42:58+0000] running backup backup-oss-native
+ yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/demand-backup/conf/backup-oss-native.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Yjs1ySKGfJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Lr0gO2Nk3u
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Yjs1ySKGfJ
perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created
+ cat /tmp/tmp.Lr0gO2Nk3u
+ rm /tmp/tmp.Yjs1ySKGfJ /tmp/tmp.Lr0gO2Nk3u
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ '[' -z '' ']'
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state................................................OK
+ wait_backup backup-gcp-cs-s3
+ local backup_name=backup-gcp-cs-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-s3 to reach ready state......OK
+ wait_backup backup-gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa to reach ready state.	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:12.225Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483", "previous": "waiting", "current": "error"}
2026-07-22T13:44:12.225Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "4271d8ff-44c1-46be-b651-949ba59a7cab", "replset": "rs0", "status": "initializing"}
2026-07-22T13:44:12.250Z	INFO	Releasing backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:12.250Z	ERROR	failed to release the lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483", "error": "not the holder", "errorVerbose": "not the holder\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.init\n\t<autogenerated>:1\nruntime.doInit1\n\t/usr/local/go/src/runtime/proc.go:8103\nruntime.doInit\n\t/usr/local/go/src/runtime/proc.go:8070\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:258\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:171
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:250
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:12.250Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483"}
2026-07-22T13:44:12.250Z	INFO	Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483"}
2026-07-22T13:44:12.250Z	ERROR	Reconciler error	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "4ee68f01-7844-41d2-a12b-1a09355cc483", "error": "reconcile backup: check for concurrent jobs: getting PBM object: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017: create mongo connection: ping: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local on 34.118.224.10:53: read udp 10.172.32.11:60303->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: RSSecondary, Tag sets: nodeName=gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w,region=us-central1,zone=us-central1-a,serviceName=some-name,podName=some-name-rs0-2, Average RTT: 2091835 }, ] }", "errorVerbose": "server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local on 34.118.224.10:53: read udp 10.172.32.11:60303->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: RSSecondary, Tag sets: nodeName=gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w,region=us-central1,zone=us-central1-a,serviceName=some-name,podName=some-name-rs0-2, Average RTT: 2091835 }, ] }\nping\ncreate mongo connection\ncreate PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:234\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.HasActiveJobs\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/backup.go:66\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:271\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ngetting PBM object\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.HasActiveJobs\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/backup.go:71\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:271\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ncheck for concurrent jobs\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:273\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:246\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:12.251Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88"}
2026-07-22T13:44:12.266Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88", "state": "waiting"}
2026-07-22T13:44:12.266Z	ERROR	Reconciler error	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "4271d8ff-44c1-46be-b651-949ba59a7cab", "error": "reconcile statefulsets: create system users: failed to get mongo client: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,nodeName=gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w,serviceName=some-name,podName=some-name-rs0-2,region=us-central1, Average RTT: 2028716 }, ] }", "errorVerbose": "create system users: failed to get mongo client: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,nodeName=gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w,serviceName=some-name,podName=some-name-rs0-2,region=us-central1, Average RTT: 2028716 }, ] }\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:489\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:22.301Z	ERROR	FULL CLUSTER CRASH	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "error": "ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, ] }", "errorVerbose": "server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:182\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongoClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:26\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongo\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:38\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongoClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:60\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:129\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:627\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:487\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:136
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:627
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:487
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:22.302Z	INFO	handleReplicaSetNoPrimary	Connecting to pod	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "pod": "some-name-rs0-0", "user": "clusterAdmin"}
2026-07-22T13:44:32.236Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "cfg": {"_id":"rs0","version":8,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2474-67718-default-pool-52010616-x273","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a60c53dea8c0f793e2f3a6e"},"writeConcernMajorityJournalDefault":true}}
2026-07-22T13:44:32.241Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "replset": "rs0", "status": "error"}
2026-07-22T13:44:32.290Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:32.290Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:32.347Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:32.347Z	INFO	Another backup is holding the lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:32.349Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "40636c17-2259-430e-97e3-b2f74db83b88"}
2026-07-22T13:44:32.349Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "4aee431d-15ab-46d1-be62-c850c1e99d30"}
2026-07-22T13:44:32.370Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "4aee431d-15ab-46d1-be62-c850c1e99d30", "state": "running"}
2026-07-22T13:44:32.423Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "error"}
2026-07-22T13:44:32.423Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "679009ab-400e-4427-ad37-648fcabc6aa1", "previous": "ready", "current": "initializing"}
2026-07-22T13:44:32.434Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "4aee431d-15ab-46d1-be62-c850c1e99d30", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:32.435Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "4aee431d-15ab-46d1-be62-c850c1e99d30", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1823\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:266\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:267
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:32.436Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "4aee431d-15ab-46d1-be62-c850c1e99d30"}
2026-07-22T13:44:32.437Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "3343e1fd-e3ae-4c21-b2e0-8c9928c1a55c"}
2026-07-22T13:44:32.560Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "3343e1fd-e3ae-4c21-b2e0-8c9928c1a55c", "state": "waiting"}
2026-07-22T13:44:32.602Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "3343e1fd-e3ae-4c21-b2e0-8c9928c1a55c", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:32.602Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "3343e1fd-e3ae-4c21-b2e0-8c9928c1a55c", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1823\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:266\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:267
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:244
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312
2026-07-22T13:44:32.603Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "3343e1fd-e3ae-4c21-b2e0-8c9928c1a55c"}
2026-07-22T13:44:32.604Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "fcd914b1-a54b-4963-b865-dde9757a3a55"}
2026-07-22T13:44:32.760Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "fcd914b1-a54b-4963-b865-dde9757a3a55", "state": "error"}
2026-07-22T13:44:32.760Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sa","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-sa", "reconcileID": "fcd914b1-a54b-4963-b865-dde9757a3a55"}
2026-07-22T13:44:33.034Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "6c7f68ba-0457-4031-be93-184dfcfc6e0f", "previous": "error", "current": "ready"}
2026-07-22T13:44:33.411Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "some-name", "reconcileID": "e6eb451c-b94d-4525-a168-5595d7f606ca", "previous": "initializing", "current": "ready"}
2026-07-22T13:44:37.350Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71"}
2026-07-22T13:44:37.370Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71", "state": "waiting"}
2026-07-22T13:44:37.416Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:37.416Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:37.460Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:37.461Z	INFO	Another backup is holding the lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:37.462Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "47b5c745-299b-47a5-93fa-ba5d94feaa71"}
2026-07-22T13:44:37.463Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d"}
2026-07-22T13:44:37.476Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "state": "running"}
2026-07-22T13:44:37.523Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:37.523Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "currentJob": {"Name":"backup-aws-s3","Type":0}}
2026-07-22T13:44:37.582Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:37.584Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "backup": "backup-aws-s3", "pbmName": "2026-07-22T13:43:22Z", "meta": {"type":"logical","opid":"6a60c8fa37884eebada1daa1","name":"2026-07-22T13:43:22Z","replsets":[{"name":"rs0","backup_name":"2026-07-22T13:43:22Z/rs0/metadata.json","oplog_name":"2026-07-22T13:43:22Z/rs0/oplog","start_ts":1784727804,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1784727809,"first_write_ts":{"T":1784727804,"I":4},"last_write_ts":{"T":1784727809,"I":1},"node":"some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","conditions":[{"timestamp":1784727804,"status":"running"},{"timestamp":1784727807,"status":"dumpDone"},{"timestamp":1784727809,"status":"done"}],"pbm_version":"2.15.0","mongo_version":"8.0.26-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup","credentials":{"access-key-id":"***","secret-access-key":"***","vault":{}},"maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":87540,"size_uncompressed":0,"mongodb_version":"8.0.26-11","fcv":"8.0","start_ts":1784727803,"last_transition_ts":1784727811,"first_write_ts":{"T":1784727804,"I":4},"last_write_ts":{"T":1784727809,"I":1},"hb":{"T":1784727809,"I":11},"status":"done","conditions":[{"timestamp":1784727803,"status":"starting"},{"timestamp":1784727806,"status":"running"},{"timestamp":1784727809,"status":"dumpDone"},{"timestamp":1784727811,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"}],"pbm_version":"2.15.0","balancer":"off"}}
2026-07-22T13:44:37.587Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "previous": "running", "current": "ready"}
2026-07-22T13:44:37.609Z	INFO	Releasing backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:37.649Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "83d924a7-4130-48d8-9a6c-11b58ba44b7d"}
2026-07-22T13:44:37.649Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b"}
2026-07-22T13:44:37.682Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "state": "waiting"}
2026-07-22T13:44:37.729Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:37.729Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "currentJob": {"Name":"backup-gcp-cs-s3","Type":0}}
2026-07-22T13:44:37.784Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:37.851Z	DEBUG	Added finalizer	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "finalizer": "internal.percona.com/release-lock"}
2026-07-22T13:44:37.851Z	INFO	Starting backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "backup": "backup-gcp-cs-s3", "storage": "gcp-cs-s3"}
2026-07-22T13:44:37.851Z	INFO	Sending backup command	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "backup": "backup-gcp-cs-s3", "storage": "gcp-cs-s3", "backupCmd": "backup [name: 2026-07-22T13:44:37Z, compression: gzip (level: default)] <ts: 0>", "profile": "gcp-cs-s3"}
2026-07-22T13:44:37.858Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b", "previous": "waiting", "current": "requested"}
2026-07-22T13:44:37.885Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "c447b37d-ad60-4e64-bac9-a5758fc25d1b"}
2026-07-22T13:44:37.886Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "7dc28fd6-ced2-40bf-8e67-228af94b0e20"}
2026-07-22T13:44:37.902Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "7dc28fd6-ced2-40bf-8e67-228af94b0e20", "state": "ready"}
2026-07-22T13:44:37.902Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-aws-s3", "reconcileID": "7dc28fd6-ced2-40bf-8e67-228af94b0e20"}
2026-07-22T13:44:37.902Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95"}
2026-07-22T13:44:37.919Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95", "state": "requested"}
2026-07-22T13:44:38.108Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:38.108Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95", "currentJob": {"Name":"backup-gcp-cs-s3","Type":0}}
2026-07-22T13:44:38.155Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:38.157Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95", "pbmName": "2026-07-22T13:44:37Z", "backup": "backup-gcp-cs-s3"}
2026-07-22T13:44:38.158Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "b8f569d3-6ccd-4bbe-9ee5-ca8c8edc0f95"}
2026-07-22T13:44:42.463Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04"}
2026-07-22T13:44:42.498Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04", "state": "waiting"}
2026-07-22T13:44:42.565Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:42.565Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:42.620Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:42.620Z	INFO	Another backup is holding the lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:42.622Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "bbc4dc35-669e-403b-a29f-f815b6590b04"}
2026-07-22T13:44:43.159Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572"}
2026-07-22T13:44:43.189Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "state": "requested"}
2026-07-22T13:44:43.242Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:43.242Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "currentJob": {"Name":"backup-gcp-cs-s3","Type":0}}
2026-07-22T13:44:43.293Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:43.295Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "backup": "backup-gcp-cs-s3", "pbmName": "2026-07-22T13:44:37Z", "meta": {"type":"logical","opid":"6a60c94537884eebada1db0b","name":"2026-07-22T13:44:37Z","replsets":[{"name":"rs0","backup_name":"2026-07-22T13:44:37Z/rs0/metadata.json","oplog_name":"2026-07-22T13:44:37Z/rs0/oplog","start_ts":1784727878,"status":"dumpDone","size":0,"size_uncompressed":0,"last_transition_ts":1784727882,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","conditions":[{"timestamp":1784727878,"status":"running"},{"timestamp":1784727882,"status":"dumpDone"}],"pbm_version":"2.15.0","mongo_version":"8.0.26-11"}],"compression":"gzip","store":{"name":"gcp-cs-s3","profile":true,"type":"gcs","gcs":{"bucket":"operator-testing","prefix":"psmdb-demand-backup","chunkSize":10485760,"retryer":{"backoffInitial":1000000000,"backoffMax":30000000000,"backoffMultiplier":2,"maxAttempts":5,"chunkRetryDeadline":32000000000}}},"size":0,"size_uncompressed":0,"mongodb_version":"8.0.26-11","fcv":"8.0","start_ts":1784727878,"last_transition_ts":1784727880,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":1,"I":1},"hb":{"T":1784727878,"I":15},"status":"running","conditions":[{"timestamp":1784727878,"status":"starting"},{"timestamp":1784727880,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"],"ack":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"}],"pbm_version":"2.15.0","balancer":"off"}}
2026-07-22T13:44:43.299Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572", "previous": "requested", "current": "running"}
2026-07-22T13:44:43.348Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "2a7ceae8-730f-467b-b422-00411aa4f572"}
2026-07-22T13:44:43.355Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c"}
2026-07-22T13:44:43.382Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c", "state": "running"}
2026-07-22T13:44:43.432Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:43.432Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c", "currentJob": {"Name":"backup-gcp-cs-s3","Type":0}}
2026-07-22T13:44:43.486Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:43.491Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c", "backup": "backup-gcp-cs-s3", "pbmName": "2026-07-22T13:44:37Z", "meta": {"type":"logical","opid":"6a60c94537884eebada1db0b","name":"2026-07-22T13:44:37Z","replsets":[{"name":"rs0","backup_name":"2026-07-22T13:44:37Z/rs0/metadata.json","oplog_name":"2026-07-22T13:44:37Z/rs0/oplog","start_ts":1784727878,"status":"dumpDone","size":0,"size_uncompressed":0,"last_transition_ts":1784727882,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","conditions":[{"timestamp":1784727878,"status":"running"},{"timestamp":1784727882,"status":"dumpDone"}],"pbm_version":"2.15.0","mongo_version":"8.0.26-11"}],"compression":"gzip","store":{"name":"gcp-cs-s3","profile":true,"type":"gcs","gcs":{"bucket":"operator-testing","prefix":"psmdb-demand-backup","chunkSize":10485760,"retryer":{"backoffInitial":1000000000,"backoffMax":30000000000,"backoffMultiplier":2,"maxAttempts":5,"chunkRetryDeadline":32000000000}}},"size":0,"size_uncompressed":0,"mongodb_version":"8.0.26-11","fcv":"8.0","start_ts":1784727878,"last_transition_ts":1784727880,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":1,"I":1},"hb":{"T":1784727878,"I":15},"status":"running","conditions":[{"timestamp":1784727878,"status":"starting"},{"timestamp":1784727880,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"],"ack":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"}],"pbm_version":"2.15.0","balancer":"off"}}
2026-07-22T13:44:43.495Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "7cd0c6d3-b2d9-4d9a-a910-1ecf661bf77c"}
2026-07-22T13:44:47.622Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0"}
2026-07-22T13:44:47.639Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0", "state": "waiting"}
2026-07-22T13:44:47.690Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:47.690Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:47.736Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:47.736Z	INFO	Another backup is holding the lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:47.738Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "962eec4d-b20f-4cfe-864d-704c233492f0"}
2026-07-22T13:44:48.496Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa"}
2026-07-22T13:44:48.508Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "state": "running"}
2026-07-22T13:44:48.563Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:48.563Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "currentJob": {"Name":"backup-gcp-cs-s3","Type":0}}
2026-07-22T13:44:48.614Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:48.616Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "backup": "backup-gcp-cs-s3", "pbmName": "2026-07-22T13:44:37Z", "meta": {"type":"logical","opid":"6a60c94537884eebada1db0b","name":"2026-07-22T13:44:37Z","replsets":[{"name":"rs0","backup_name":"2026-07-22T13:44:37Z/rs0/metadata.json","oplog_name":"2026-07-22T13:44:37Z/rs0/oplog","start_ts":1784727878,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1784727884,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":1784727883,"I":5},"node":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","conditions":[{"timestamp":1784727878,"status":"running"},{"timestamp":1784727882,"status":"dumpDone"},{"timestamp":1784727884,"status":"done"}],"pbm_version":"2.15.0","mongo_version":"8.0.26-11"}],"compression":"gzip","store":{"name":"gcp-cs-s3","profile":true,"type":"gcs","gcs":{"bucket":"operator-testing","prefix":"psmdb-demand-backup","chunkSize":10485760,"retryer":{"backoffInitial":1000000000,"backoffMax":30000000000,"backoffMultiplier":2,"maxAttempts":5,"chunkRetryDeadline":32000000000}}},"size":91829,"size_uncompressed":0,"mongodb_version":"8.0.26-11","fcv":"8.0","start_ts":1784727878,"last_transition_ts":1784727885,"first_write_ts":{"T":1784727878,"I":15},"last_write_ts":{"T":1784727883,"I":5},"hb":{"T":1784727883,"I":2},"status":"done","conditions":[{"timestamp":1784727878,"status":"starting"},{"timestamp":1784727880,"status":"running"},{"timestamp":1784727883,"status":"dumpDone"},{"timestamp":1784727885,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"],"ack":"some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"}],"pbm_version":"2.15.0","balancer":"off"}}
2026-07-22T13:44:48.619Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "previous": "running", "current": "ready"}
2026-07-22T13:44:48.646Z	INFO	Releasing backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:48.690Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "24fe16c7-3a70-4021-8def-d643fb0aecaa"}
2026-07-22T13:44:48.690Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "8e1e6f29-12ae-4079-88b1-6ef3108871c7"}
2026-07-22T13:44:48.703Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "8e1e6f29-12ae-4079-88b1-6ef3108871c7", "state": "ready"}
2026-07-22T13:44:48.703Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-gcp-cs-s3", "reconcileID": "8e1e6f29-12ae-4079-88b1-6ef3108871c7"}
2026-07-22T13:44:52.739Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc"}
2026-07-22T13:44:52.783Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "state": "waiting"}
2026-07-22T13:44:52.830Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:52.830Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:52.876Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:52.951Z	DEBUG	Added finalizer	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "finalizer": "internal.percona.com/release-lock"}
2026-07-22T13:44:52.951Z	INFO	Starting backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "backup": "backup-azure-blob", "storage": "azure-blob"}
2026-07-22T13:44:52.951Z	INFO	Sending backup command	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "backup": "backup-azure-blob", "storage": "azure-blob", "backupCmd": "backup [name: 2026-07-22T13:44:52Z, compression: gzip (level: default)] <ts: 0>", "profile": "azure-blob"}
2026-07-22T13:44:52.995Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc", "previous": "waiting", "current": "requested"}
2026-07-22T13:44:53.032Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "22fd0db9-cf2b-4694-a0d8-8db8d7fbeedc"}
2026-07-22T13:44:53.033Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af"}
2026-07-22T13:44:53.051Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af", "state": "requested"}
2026-07-22T13:44:53.132Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:53.132Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:53.224Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:53.226Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af", "pbmName": "2026-07-22T13:44:52Z", "backup": "backup-azure-blob"}
2026-07-22T13:44:53.227Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "27b0ee9c-09bb-445a-a047-31c407fb40af"}
2026-07-22T13:44:53.227Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1"}
2026-07-22T13:44:53.256Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1", "state": "requested"}
2026-07-22T13:44:53.298Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1", "cluster": "some-name", "namespace": "demand-backup-7490"}
2026-07-22T13:44:53.298Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1", "currentJob": {"Name":"backup-azure-blob","Type":0}}
2026-07-22T13:44:53.349Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1", "lease": "psmdb-some-name-backup-lock"}
2026-07-22T13:44:53.351Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1", "backup": "backup-azure-blob", "pbmName": "2026-07-22T13:44:52Z", "meta": {"type":"logical","opid":"6a60c95437884eebada1dbc9","name":"2026-07-22T13:44:52Z","replsets":[],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","endpointUrl":"https://engk8soperators.blob.core.windows.net","prefix":"psmdb-demand-backup","credentials":{"key":"***"},"retryer":{"numMaxRetries":3,"minRetryDelay":800000000,"maxRetryDelay":60000000000}}},"size":0,"size_uncompressed":0,"mongodb_version":"8.0.26-11","fcv":"8.0","start_ts":1784727893,"last_transition_ts":1784727893,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1784727893,"I":2},"status":"starting","conditions":[{"timestamp":1784727893,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.15.0","balancer":"off"}}
2026-07-22T13:44:53.395Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-7490"}, "namespace": "demand-backup-7490", "name": "backup-azure-blob", "reconcileID": "23faf874-7987-4469-9fa3-1903c5e0b8f1"}
NAME                CLUSTER     STORAGE      DESTINATION                                                                                               TYPE      SIZE      STATUS      COMPLETED   AGE
backup-aws-s3       some-name   aws-s3       s3://operator-testing/psmdb-demand-backup/2026-07-22T13:43:22Z                                            logical   85.49KB   ready       84s         2m4s
backup-azure-blob   some-name   azure-blob   https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-22T13:44:52Z                       requested               117s
backup-gcp-cs-s3    some-name   gcp-cs-s3    s3://operator-testing/psmdb-demand-backup/2026-07-22T13:44:37Z                                            logical   89.68KB   ready       10s         2m2s
backup-gcp-cs-sa    some-name   gcp-cs-sa                                                                                                                                  error                   2m
backup-minio        some-name   minio        s3://operator-testing/2026-07-22T13:42:49Z                                                                logical   82.70KB   ready       2m          2m6s
backup-oss-native   some-name   oss-native   oss://operator-testing/psmdb-demand-backup-oss-native/2026-07-22T13:43:01Z                                logical   89.75KB   ready       96s         115s
Name:         backup-gcp-cs-sa
Namespace:    demand-backup-7490
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBBackup
Metadata:
  Creation Timestamp:  2026-07-22T13:42:55Z
  Finalizers:
    percona.com/delete-backup
  Generation:        1
  Resource Version:  1784727852241711014
  UID:               220d5189-cf82-4265-8d53-87b65a14f67c
Spec:
  Cluster Name:               some-name
  Starting Deadline Seconds:  120
  Storage Name:               gcp-cs-sa
  Type:                       logical
Status:
  Error:  check for concurrent jobs: getting PBM object: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017: create mongo connection: ping: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-7490.svc.cluster.local on 34.118.224.10:53: read udp 10.172.32.11:60303->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: Unknown }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-7490.svc.cluster.local:27017, Type: RSSecondary, Tag sets: nodeName=gke-jen-psmdb-2474-67718-default-pool-52010616-5l4w,region=us-central1,zone=us-central1-a,serviceName=some-name,podName=some-name-rs0-2, Average RTT: 2091835 }, ] }
  State:  error
Events:   <none>
Backup object psmdb-backup/backup-gcp-cs-sa is in error state.