Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/logs/demand-backup-snapshot-vault.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ vault_name=vault-service
++ deploy_volume_snapshot_class
++ local snapshot_class_file
++ local k8s_env
+++ detect_k8s_env
+++ local provider=unknown
+++ grep -q openshift.io
+++ kubectl_bin api-resources
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V4hiR9iFdz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.taxT1HY7xZ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in $(seq 0 2)
+++ set +e
+++ kubectl api-resources
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.V4hiR9iFdz
+++ cat /tmp/tmp.taxT1HY7xZ
+++ rm /tmp/tmp.V4hiR9iFdz /tmp/tmp.taxT1HY7xZ
+++ return 0
+++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}'
+++ grep -q eks.amazonaws.com
+++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}'
+++ grep -q cloud.google.com/gke
+++ provider=gke
+++ echo gke
++ k8s_env=gke
++ case "$k8s_env" in
++ snapshot_class_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/gke-volume-snapshot-class.yaml
++ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/gke-volume-snapshot-class.yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DUIQQuzHf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YoypcYXVFX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/gke-volume-snapshot-class.yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DUIQQuzHf0
volumesnapshotclass.snapshot.storage.k8s.io/gke-snapshot-class unchanged
++ cat /tmp/tmp.YoypcYXVFX
++ rm /tmp/tmp.DUIQQuzHf0 /tmp/tmp.YoypcYXVFX
++ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/gke-volume-snapshot-class.yaml
+ VOLUME_SNAPSHOT_CLASS=gke-snapshot-class
+ cluster=some-name
+ create_infra demand-backup-snapshot-vault-16999
+ local ns=demand-backup-snapshot-vault-16999
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.qZWxkROIZ3
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZLZUYVlWcu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qZWxkROIZ3
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.ZLZUYVlWcu
+ rm /tmp/tmp.qZWxkROIZ3 /tmp/tmp.ZLZUYVlWcu
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gGGxVZkIDf
++ mktemp
+ local LAST_ERR=/tmp/tmp.ekpATpRZBZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gGGxVZkIDf
+ cat /tmp/tmp.ekpATpRZBZ
+ rm /tmp/tmp.gGGxVZkIDf /tmp/tmp.ekpATpRZBZ
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.hH4CVL0KN9
++ mktemp
+ local LAST_ERR=/tmp/tmp.5ufsYC01YG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hH4CVL0KN9
+ cat /tmp/tmp.5ufsYC01YG
+ rm /tmp/tmp.hH4CVL0KN9 /tmp/tmp.5ufsYC01YG
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.mlGsxTJxpu
++ mktemp
+ local LAST_ERR=/tmp/tmp.ze8ErAdcBv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mlGsxTJxpu
+ cat /tmp/tmp.ze8ErAdcBv
+ rm /tmp/tmp.mlGsxTJxpu /tmp/tmp.ze8ErAdcBv
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.GrrmvQwAJl
++ mktemp
+ local LAST_ERR=/tmp/tmp.m1KxLYiRXR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GrrmvQwAJl
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.m1KxLYiRXR
+ rm /tmp/tmp.GrrmvQwAJl /tmp/tmp.m1KxLYiRXR
+ return 0
+ check_crd_for_deletion PR-2283-73ca3795
+ local git_tag=PR-2283-73ca3795
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2283-73ca3795/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jS5XZVBM7p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SzhxFg0z0N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jS5XZVBM7p
++ cat /tmp/tmp.SzhxFg0z0N
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jS5XZVBM7p
++ cat /tmp/tmp.SzhxFg0z0N
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jS5XZVBM7p
++ cat /tmp/tmp.SzhxFg0z0N
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.jS5XZVBM7p
++ cat /tmp/tmp.SzhxFg0z0N
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.jS5XZVBM7p /tmp/tmp.SzhxFg0z0N
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ntBQKrRiq
+ local LAST_OUT=/tmp/tmp.mZMYXun7Sl
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Wuruxl6k8
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.WPKdLRgvbx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mZMYXun7Sl
+ cat /tmp/tmp.WPKdLRgvbx
+ rm /tmp/tmp.mZMYXun7Sl /tmp/tmp.WPKdLRgvbx
+ return 0
namespace "demand-backup-snapshot-vault-7230" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8ntBQKrRiq
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.3Wuruxl6k8
+ rm /tmp/tmp.8ntBQKrRiq /tmp/tmp.3Wuruxl6k8
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wa6rGT2Bri
++ mktemp
+ local LAST_ERR=/tmp/tmp.GmE2hNjbvx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wa6rGT2Bri
+ cat /tmp/tmp.GmE2hNjbvx
+ rm /tmp/tmp.wa6rGT2Bri /tmp/tmp.GmE2hNjbvx
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.iWdvDiX7hg
++ mktemp
+ local LAST_ERR=/tmp/tmp.aSVwqwBDK2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iWdvDiX7hg
namespace/psmdb-operator created
+ cat /tmp/tmp.aSVwqwBDK2
+ rm /tmp/tmp.iWdvDiX7hg /tmp/tmp.aSVwqwBDK2
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rUSUsMr8Iq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r4BqDaG4mb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rUSUsMr8Iq
++ cat /tmp/tmp.r4BqDaG4mb
++ rm /tmp/tmp.rUSUsMr8Iq /tmp/tmp.r4BqDaG4mb
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.lapHt2ZIOE
++ mktemp
+ local LAST_ERR=/tmp/tmp.PmQCEfLbyL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lapHt2ZIOE
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11" modified.
+ cat /tmp/tmp.PmQCEfLbyL
+ rm /tmp/tmp.lapHt2ZIOE /tmp/tmp.PmQCEfLbyL
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uupTalGlHr
++ mktemp
+ local LAST_ERR=/tmp/tmp.WSd4HwUT7z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uupTalGlHr
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.WSd4HwUT7z
+ rm /tmp/tmp.uupTalGlHr /tmp/tmp.WSd4HwUT7z
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KtWarXJ6Uy
++ mktemp
+ local LAST_ERR=/tmp/tmp.2FB7sPQxAV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KtWarXJ6Uy
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.2FB7sPQxAV
+ rm /tmp/tmp.KtWarXJ6Uy /tmp/tmp.2FB7sPQxAV
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XP9T616Tw1
++ mktemp
+ local LAST_ERR=/tmp/tmp.LH6liBfeqR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XP9T616Tw1
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.LH6liBfeqR
+ rm /tmp/tmp.XP9T616Tw1 /tmp/tmp.LH6liBfeqR
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9XCBBmtbcW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8ifWCE0USC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9XCBBmtbcW
++ cat /tmp/tmp.8ifWCE0USC
++ rm /tmp/tmp.9XCBBmtbcW /tmp/tmp.8ifWCE0USC
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-79d56c85d6-45s2g
+ local pod=percona-server-mongodb-operator-79d56c85d6-45s2g
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-79d56c85d6-45s2g to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EDjxpvqeM3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VYLyItGx9J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EDjxpvqeM3
++ cat /tmp/tmp.VYLyItGx9J
++ rm /tmp/tmp.EDjxpvqeM3 /tmp/tmp.VYLyItGx9J
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-79d56c85d6-45s2g
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y48zkf6PLm
++ mktemp
+ local LAST_ERR=/tmp/tmp.w3qfuLwKZt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-79d56c85d6-45s2g
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y48zkf6PLm
+ cat /tmp/tmp.w3qfuLwKZt
+ rm /tmp/tmp.Y48zkf6PLm /tmp/tmp.w3qfuLwKZt
+ return 0
2026-04-09T16:06:40.088Z	INFO	setup	Manager starting up	{"gitCommit": "73ca37956356b4f3579a3ef1dce0c4105fda2da2", "gitBranch": "PR-2283-73ca3795", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-snapshot-vault-16999
+ local namespace=demand-backup-snapshot-vault-16999
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-snapshot-vault-16999'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-snapshot-vault-16999
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-snapshot-vault-16999 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZmFDdoSeFK
++ mktemp
+ local LAST_OUT=/tmp/tmp.4mWYuC8E0b
++ mktemp
+ local LAST_ERR=/tmp/tmp.PCvjl2b2Sk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.zTr0ORoXa6
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-snapshot-vault-16999 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZmFDdoSeFK
+ cat /tmp/tmp.PCvjl2b2Sk
+ rm /tmp/tmp.ZmFDdoSeFK /tmp/tmp.PCvjl2b2Sk
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4mWYuC8E0b
+ cat /tmp/tmp.zTr0ORoXa6
+ rm /tmp/tmp.4mWYuC8E0b /tmp/tmp.zTr0ORoXa6
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-snapshot-vault-16999
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZkxpikbmV1
++ mktemp
+ local LAST_ERR=/tmp/tmp.JneinWPWXI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-snapshot-vault-16999
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZkxpikbmV1
+ cat /tmp/tmp.JneinWPWXI
+ rm /tmp/tmp.ZkxpikbmV1 /tmp/tmp.JneinWPWXI
+ return 0
+ desc 'create namespace demand-backup-snapshot-vault-16999'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-snapshot-vault-16999
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-snapshot-vault-16999
++ mktemp
+ local LAST_OUT=/tmp/tmp.BOLD9C1osX
++ mktemp
+ local LAST_ERR=/tmp/tmp.2oLk9b7zVj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-snapshot-vault-16999
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BOLD9C1osX
namespace/demand-backup-snapshot-vault-16999 created
+ cat /tmp/tmp.2oLk9b7zVj
+ rm /tmp/tmp.BOLD9C1osX /tmp/tmp.2oLk9b7zVj
+ return 0
+ set_kube_ctx demand-backup-snapshot-vault-16999
+ local namespace=demand-backup-snapshot-vault-16999
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AWAYRG7vEa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8K7JanZvGK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AWAYRG7vEa
++ cat /tmp/tmp.8K7JanZvGK
++ rm /tmp/tmp.AWAYRG7vEa /tmp/tmp.8K7JanZvGK
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11 --namespace=demand-backup-snapshot-vault-16999
++ mktemp
+ local LAST_OUT=/tmp/tmp.zPLXG3qbzA
++ mktemp
+ local LAST_ERR=/tmp/tmp.OWifOZuVBH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11 --namespace=demand-backup-snapshot-vault-16999
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zPLXG3qbzA
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster11" modified.
+ cat /tmp/tmp.OWifOZuVBH
+ rm /tmp/tmp.zPLXG3qbzA /tmp/tmp.OWifOZuVBH
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Apr  9 16:07:15 2026
NAMESPACE: demand-backup-snapshot-vault-16999
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-snapshot-vault-16999.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-snapshot-vault-16999 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-snapshot-vault-16999

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-snapshot-vault-16999 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-snapshot-vault-16999 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XbPIB55WOV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YUDWpFlYhS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XbPIB55WOV
++ cat /tmp/tmp.YUDWpFlYhS
++ rm /tmp/tmp.XbPIB55WOV /tmp/tmp.YUDWpFlYhS
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-ltgmr
+ wait_pod minio-service-6d5f646cdc-ltgmr
+ local pod=minio-service-6d5f646cdc-ltgmr
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-ltgmr to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-snapshot-vault-16999.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.lEVnaAejEx
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ta4TSkB0Yl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lEVnaAejEx
pod "aws-cli" deleted from demand-backup-snapshot-vault-16999 namespace
+ cat /tmp/tmp.Ta4TSkB0Yl
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.lEVnaAejEx /tmp/tmp.Ta4TSkB0Yl
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SzvzBGLjZc
++ mktemp
+ local LAST_ERR=/tmp/tmp.932D30nuVe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SzvzBGLjZc
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.932D30nuVe
+ rm /tmp/tmp.SzvzBGLjZc /tmp/tmp.932D30nuVe
+ return 0
+ desc 'Setting up Vault'
+ set +o xtrace


-----------------------------------------------------------------------------------
Setting up Vault
-----------------------------------------------------------------------------------

+ setup_vault
+ local sa_namespace=demand-backup-snapshot-vault-16999
+ '[' -n psmdb-operator ']'
+ sa_namespace=psmdb-operator
+ deploy_vault vault-service --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true
+ local name=vault-service
+ [[ 7 -gt 0 ]]
+ shift
+ desc 'install Vault vault-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Vault vault-service
-----------------------------------------------------------------------------------

+ helm uninstall vault-service
Error: uninstall: Release not loaded: vault-service: release: not found
+ :
+ helm repo remove hashicorp
"hashicorp" has been removed from your repositories
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" has been added to your repositories
+ destroy_vault vault-service
+ local name=vault-service
+ local vault_ns
++ helm list --all-namespaces --filter vault-service
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ vault_ns=
+ desc 'destroy vault'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy vault
-----------------------------------------------------------------------------------

++ kubectl api-resources
++ grep vault
++ awk '{print $1}'
+ '[' -n '' ']'
++ kubectl get clusterrolebinding -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding
clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted
clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted
++ kubectl get clusterrole -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole
clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted
++ kubectl get mutatingwebhookconfiguration -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg
mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted
+ [[ -n '' ]]
+ retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true
NAME: vault-service
LAST DEPLOYED: Thu Apr  9 16:08:12 2026
NAMESPACE: demand-backup-snapshot-vault-16999
STATUS: deployed
REVISION: 1
NOTES:
Thank you for installing HashiCorp Vault!

Now that you have deployed Vault, you should look over the docs on using
Vault with Kubernetes available here:

https://developer.hashicorp.com/vault/docs


Your release is named vault-service. To learn more about the release, try:

  $ helm status vault-service
  $ helm get manifest vault-service
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
Running
+ sleep 5
+ kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json
++ mktemp
+ local LAST_OUT=/tmp/tmp.GhKqfyjdDE
++ mktemp
+ local LAST_ERR=/tmp/tmp.KFB1QClk3k
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GhKqfyjdDE
+ cat /tmp/tmp.KFB1QClk3k
+ rm /tmp/tmp.GhKqfyjdDE /tmp/tmp.KFB1QClk3k
+ return 0
+ local unsealKey
++ jq -r '.unseal_keys_b64[]'
+ unsealKey=m8oc+t6tytmnQLD8SbtvGWFo/kT1t3m+R03mBC8T2Tk=
+ local token
++ jq -r .root_token
+ token=hvs.sAsEppREjjzfRwUi0EIc0MLS
+ kubectl_bin exec pod/vault-service-0 -- vault operator unseal m8oc+t6tytmnQLD8SbtvGWFo/kT1t3m+R03mBC8T2Tk=
++ mktemp
+ local LAST_OUT=/tmp/tmp.UqI8DYyVEA
++ mktemp
+ local LAST_ERR=/tmp/tmp.AO9tAnvvmh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec pod/vault-service-0 -- vault operator unseal m8oc+t6tytmnQLD8SbtvGWFo/kT1t3m+R03mBC8T2Tk=
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UqI8DYyVEA
Key             Value
---             -----
Seal Type       shamir
Initialized     true
Sealed          false
Total Shares    1
Threshold       1
Version         1.21.2
Build Date      2026-01-06T08:33:05Z
Storage Type    file
Cluster Name    vault-cluster-06c76547
Cluster ID      f45cfda6-9c4b-3d38-04b5-9e964678ff24
HA Enabled      false
+ cat /tmp/tmp.AO9tAnvvmh
+ rm /tmp/tmp.UqI8DYyVEA /tmp/tmp.AO9tAnvvmh
+ return 0
+ kubectl_bin exec -it pod/vault-service-0 -- sh
++ mktemp
+ local LAST_OUT=/tmp/tmp.swKinJiAe7
++ mktemp
+ local LAST_ERR=/tmp/tmp.jyWu9uvOi2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec -it pod/vault-service-0 -- sh
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.swKinJiAe7
Success! You are now authenticated. The token information displayed below
is already stored in the token helper. You do NOT need to run "vault login"
again. Future Vault requests will automatically use this token.

Key                  Value
---                  -----
token                hvs.sAsEppREjjzfRwUi0EIc0MLS
token_accessor       o6dayWXs8i4l0JwYrbb5ZfY6
token_duration       ∞
token_renewable      false
token_policies       ["root"]
identity_policies    []
policies             ["root"]
Success! Enabled the kv-v2 secrets engine at: secret/
+ cat /tmp/tmp.jyWu9uvOi2
Unable to use a TTY - input is not a terminal or the right kind of file
+ rm /tmp/tmp.swKinJiAe7 /tmp/tmp.jyWu9uvOi2
+ return 0
+ kubectl_bin create secret generic vault-secret --from-literal=token=hvs.sAsEppREjjzfRwUi0EIc0MLS
++ mktemp
+ local LAST_OUT=/tmp/tmp.04iaxubBGy
++ mktemp
+ local LAST_ERR=/tmp/tmp.9QEwKz1tvL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create secret generic vault-secret --from-literal=token=hvs.sAsEppREjjzfRwUi0EIc0MLS
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.04iaxubBGy
secret/vault-secret created
+ cat /tmp/tmp.9QEwKz1tvL
+ rm /tmp/tmp.04iaxubBGy /tmp/tmp.9QEwKz1tvL
+ return 0
+ sleep 10
+ wait_pod vault-service-0
+ local pod=vault-service-0
+ set +o xtrace
waiting for pod/vault-service-0 to be ready.OK
+ sleep 20
+ kubectl_bin exec vault-service-0 -- vault auth enable kubernetes
++ mktemp
+ local LAST_OUT=/tmp/tmp.GciAv9RpOw
++ mktemp
+ local LAST_ERR=/tmp/tmp.gbpaIeCP1m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- vault auth enable kubernetes
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GciAv9RpOw
Success! Enabled kubernetes auth method at: kubernetes/
+ cat /tmp/tmp.gbpaIeCP1m
+ rm /tmp/tmp.GciAv9RpOw /tmp/tmp.gbpaIeCP1m
+ return 0
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/role-binding.yml
+ yq '.metadata.namespace="demand-backup-snapshot-vault-16999"'
+ yq '.subjects[0].namespace="psmdb-operator"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.d3jJiq9mJz
++ mktemp
+ local LAST_ERR=/tmp/tmp.DWAH0iltuB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d3jJiq9mJz
rolebinding.rbac.authorization.k8s.io/vault-role-binding created
+ cat /tmp/tmp.DWAH0iltuB
+ rm /tmp/tmp.d3jJiq9mJz /tmp/tmp.DWAH0iltuB
+ return 0
++ kubectl_bin exec vault-service-0 -- vault token create -policy=operator -format=json
++ jq -r .auth.client_token
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yinZwTiOFD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a72w0qy4dU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec vault-service-0 -- vault token create -policy=operator -format=json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yinZwTiOFD
++ cat /tmp/tmp.a72w0qy4dU
++ rm /tmp/tmp.yinZwTiOFD /tmp/tmp.a72w0qy4dU
++ return 0
+ token=hvs.CAESIJmFKXHFngGSBmn_687uGr4802EnbLSQ-jmRFJOiXzJNGh4KHGh2cy5DS0RTTDBGQ1RpNUtEZWoyT3pxaHpPMFQ
+ kubectl_bin create secret generic vault-sync-secret --from-literal=token=hvs.CAESIJmFKXHFngGSBmn_687uGr4802EnbLSQ-jmRFJOiXzJNGh4KHGh2cy5DS0RTTDBGQ1RpNUtEZWoyT3pxaHpPMFQ
++ mktemp
+ local LAST_OUT=/tmp/tmp.zKDXbN7hG7
++ mktemp
+ local LAST_ERR=/tmp/tmp.Og82IqvjR5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create secret generic vault-sync-secret --from-literal=token=hvs.CAESIJmFKXHFngGSBmn_687uGr4802EnbLSQ-jmRFJOiXzJNGh4KHGh2cy5DS0RTTDBGQ1RpNUtEZWoyT3pxaHpPMFQ
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zKDXbN7hG7
secret/vault-sync-secret created
+ cat /tmp/tmp.Og82IqvjR5
+ rm /tmp/tmp.zKDXbN7hG7 /tmp/tmp.Og82IqvjR5
+ return 0
+ kubectl_bin exec vault-service-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT'
++ mktemp
+ local LAST_OUT=/tmp/tmp.17x1amYfsl
++ mktemp
+ local LAST_ERR=/tmp/tmp.tqT13iCJoA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.17x1amYfsl
Success! Data written to: auth/kubernetes/config
+ cat /tmp/tmp.tqT13iCJoA
+ rm /tmp/tmp.17x1amYfsl /tmp/tmp.tqT13iCJoA
+ return 0
+ kubectl_bin exec vault-service-0 -- sh -c $'vault policy write operator - <<EOF\n    # KVv1\n    path "secret/psmdb/operator/*" {\n      capabilities = ["read"]\n    }\n\n    # KVv2\n    path "secret/data/psmdb/operator/*" {\n      capabilities = ["read"]\n    }\nEOF\n'
++ mktemp
+ local LAST_OUT=/tmp/tmp.A5XR5tls9y
++ mktemp
+ local LAST_ERR=/tmp/tmp.BhVp6qcJ9S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c $'vault policy write operator - <<EOF\n    # KVv1\n    path "secret/psmdb/operator/*" {\n      capabilities = ["read"]\n    }\n\n    # KVv2\n    path "secret/data/psmdb/operator/*" {\n      capabilities = ["read"]\n    }\nEOF\n'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A5XR5tls9y
Success! Uploaded policy: operator
+ cat /tmp/tmp.BhVp6qcJ9S
+ rm /tmp/tmp.A5XR5tls9y /tmp/tmp.BhVp6qcJ9S
+ return 0
+ kubectl_bin exec vault-service-0 -- vault write auth/kubernetes/role/operator bound_service_account_names=percona-server-mongodb-operator bound_service_account_namespaces=psmdb-operator policies=operator ttl=1h
++ mktemp
+ local LAST_OUT=/tmp/tmp.P6mMQh9tjM
++ mktemp
+ local LAST_ERR=/tmp/tmp.TOb6CWmGi3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- vault write auth/kubernetes/role/operator bound_service_account_names=percona-server-mongodb-operator bound_service_account_namespaces=psmdb-operator policies=operator ttl=1h
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P6mMQh9tjM
+ cat /tmp/tmp.TOb6CWmGi3
WARNING! The following warnings were returned from Vault:

  * Role operator does not have an audience configured. While audiences are
  not required, consider specifying one if your use case would benefit from
  additional JWT claim verification.

+ rm /tmp/tmp.P6mMQh9tjM /tmp/tmp.TOb6CWmGi3
+ return 0
+ desc 'Seeding initial credentials in Vault'
+ set +o xtrace


-----------------------------------------------------------------------------------
Seeding initial credentials in Vault
-----------------------------------------------------------------------------------

+ vault_append MONGODB_USER_ADMIN_PASSWORD userAdmin123456
+ local key=MONGODB_USER_ADMIN_PASSWORD
+ local value=userAdmin123456
+ local tmp_json
++ mktemp
+ tmp_json=/tmp/tmp.AgH7DOtTPe
+ local new_tmp_json
++ mktemp
+ new_tmp_json=/tmp/tmp.H5eOxoecpo
+ kubectl_bin exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
+ jq .data.data
++ mktemp
+ local LAST_OUT=/tmp/tmp.fgl7RAmFpf
++ mktemp
+ local LAST_ERR=/tmp/tmp.2YRGnqDeeJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
+ exit_status=2
+ set -e
+ '[' 2 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.fgl7RAmFpf
+ cat /tmp/tmp.2YRGnqDeeJ
No value found at secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users
command terminated with exit code 2
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
+ exit_status=2
+ set -e
+ '[' 2 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.fgl7RAmFpf
+ cat /tmp/tmp.2YRGnqDeeJ
No value found at secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users
command terminated with exit code 2
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
+ exit_status=2
+ set -e
+ '[' 2 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.fgl7RAmFpf
+ cat /tmp/tmp.2YRGnqDeeJ
No value found at secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users
command terminated with exit code 2
+ sleep 8
+ cat /tmp/tmp.fgl7RAmFpf
+ cat /tmp/tmp.2YRGnqDeeJ
No value found at secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users
command terminated with exit code 2
+ rm /tmp/tmp.fgl7RAmFpf /tmp/tmp.2YRGnqDeeJ
+ return 2
+ '[' '!' -s /tmp/tmp.AgH7DOtTPe ']'
+ echo '{}'
+ jq --arg key MONGODB_USER_ADMIN_PASSWORD --arg value userAdmin123456 '(. // {}) + {($key): $value}' /tmp/tmp.AgH7DOtTPe
+ kubectl_bin cp /tmp/tmp.H5eOxoecpo vault-service-0:/tmp/data_new.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZLxXJ1MFdV
++ mktemp
+ local LAST_ERR=/tmp/tmp.92LAh3Yhr0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl cp /tmp/tmp.H5eOxoecpo vault-service-0:/tmp/data_new.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZLxXJ1MFdV
+ cat /tmp/tmp.92LAh3Yhr0
+ rm /tmp/tmp.ZLxXJ1MFdV /tmp/tmp.92LAh3Yhr0
+ return 0
+ kubectl_bin exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users @"/tmp/data_new.json"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8gLtnBFXXc
++ mktemp
+ local LAST_ERR=/tmp/tmp.vqPftWO3jQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users @"/tmp/data_new.json"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8gLtnBFXXc
================================ Secret Path ================================
secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users

======= Metadata =======
Key                Value
---                -----
created_time       2026-04-09T16:09:55.28439797Z
custom_metadata    <nil>
deletion_time      n/a
destroyed          false
version            1
+ cat /tmp/tmp.vqPftWO3jQ
+ rm /tmp/tmp.8gLtnBFXXc /tmp/tmp.vqPftWO3jQ
+ return 0
+ rm -f /tmp/tmp.AgH7DOtTPe /tmp/tmp.H5eOxoecpo
+ vault_append MONGODB_BACKUP_PASSWORD backup123456#
+ local key=MONGODB_BACKUP_PASSWORD
+ local value=backup123456#
+ local tmp_json
++ mktemp
+ tmp_json=/tmp/tmp.k58asf32Oc
+ local new_tmp_json
++ mktemp
+ new_tmp_json=/tmp/tmp.kBxvMoQycz
+ kubectl_bin exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
++ mktemp
+ jq .data.data
+ local LAST_OUT=/tmp/tmp.5R1QGxuV7r
++ mktemp
+ local LAST_ERR=/tmp/tmp.K0Ouorv04S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5R1QGxuV7r
+ cat /tmp/tmp.K0Ouorv04S
+ rm /tmp/tmp.5R1QGxuV7r /tmp/tmp.K0Ouorv04S
+ return 0
+ '[' '!' -s /tmp/tmp.k58asf32Oc ']'
+ jq --arg key MONGODB_BACKUP_PASSWORD --arg value backup123456# '(. // {}) + {($key): $value}' /tmp/tmp.k58asf32Oc
+ kubectl_bin cp /tmp/tmp.kBxvMoQycz vault-service-0:/tmp/data_new.json
++ mktemp
+ local LAST_OUT=/tmp/tmp.WzyXGm9pup
++ mktemp
+ local LAST_ERR=/tmp/tmp.Oo74HVunVy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl cp /tmp/tmp.kBxvMoQycz vault-service-0:/tmp/data_new.json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WzyXGm9pup
+ cat /tmp/tmp.Oo74HVunVy
+ rm /tmp/tmp.WzyXGm9pup /tmp/tmp.Oo74HVunVy
+ return 0
+ kubectl_bin exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users @"/tmp/data_new.json"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.RyaM2eCYKp
++ mktemp
+ local LAST_ERR=/tmp/tmp.dU5D1VCT4z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users @"/tmp/data_new.json"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RyaM2eCYKp
================================ Secret Path ================================
secret/data/psmdb/operator/demand-backup-snapshot-vault-16999/some-name/users

======= Metadata =======
Key                Value
---                -----
created_time       2026-04-09T16:10:00.861616481Z
custom_metadata    <nil>
deletion_time      n/a
destroyed          false
version            2
+ cat /tmp/tmp.dU5D1VCT4z
+ rm /tmp/tmp.RyaM2eCYKp /tmp/tmp.dU5D1VCT4z
+ return 0
+ rm -f /tmp/tmp.k58asf32Oc /tmp/tmp.kBxvMoQycz
+ desc 'Deploying PSMDB cluster with Vault and snapshot backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Deploying PSMDB cluster with Vault and snapshot backup
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.l6KoWdCHBi
++ mktemp
+ local LAST_ERR=/tmp/tmp.aZgNNRzSks
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l6KoWdCHBi
secret/some-users created
+ cat /tmp/tmp.aZgNNRzSks
+ rm /tmp/tmp.l6KoWdCHBi /tmp/tmp.aZgNNRzSks
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795"'
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-snapshot-vault-16999/g
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.GWbVYCeHJi
++ mktemp
+ local LAST_ERR=/tmp/tmp.v982YAcxv4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GWbVYCeHJi
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.v982YAcxv4
+ rm /tmp/tmp.GWbVYCeHJi /tmp/tmp.v982YAcxv4
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NbtwVQ2OIX
++ mktemp
+ local LAST_ERR=/tmp/tmp.t5Yatg5KTD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NbtwVQ2OIX
deployment.apps/psmdb-client created
+ cat /tmp/tmp.t5Yatg5KTD
+ rm /tmp/tmp.NbtwVQ2OIX /tmp/tmp.t5Yatg5KTD
+ return 0
+ echo 'check if all pods started'
check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3tRr7GVN1B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0Jyr9S8joN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3tRr7GVN1B
++ cat /tmp/tmp.0Jyr9S8joN
++ rm /tmp/tmp.3tRr7GVN1B /tmp/tmp.0Jyr9S8joN
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pwLvApFqE8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8Kd4AUEzK6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pwLvApFqE8
++ cat /tmp/tmp.8Kd4AUEzK6
++ rm /tmp/tmp.pwLvApFqE8 /tmp/tmp.8Kd4AUEzK6
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YWjfY9ZWDi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.epK0AvThr5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YWjfY9ZWDi
++ cat /tmp/tmp.epK0AvThr5
++ rm /tmp/tmp.YWjfY9ZWDi /tmp/tmp.epK0AvThr5
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xeTp8XzEms
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aese2qazHl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xeTp8XzEms
++ cat /tmp/tmp.aese2qazHl
++ rm /tmp/tmp.xeTp8XzEms /tmp/tmp.aese2qazHl
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ sleep 60
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.OK
+ desc 'Writing test data'
+ set +o xtrace


-----------------------------------------------------------------------------------
Writing test data
-----------------------------------------------------------------------------------

+ run_mongo_tls 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-16999
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eccrxh005C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VhRDxo0i48
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eccrxh005C
++ cat /tmp/tmp.VhRDxo0i48
++ rm /tmp/tmp.eccrxh005C /tmp/tmp.VhRDxo0i48
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9VPsDi0sm0
++ mktemp
+ local LAST_ERR=/tmp/tmp.vMrFaq1nPE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9VPsDi0sm0
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true
{"t":{"$date":"2026-04-09T16:12:58.908Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("d51e6d23-c7d6-4fec-8357-7948d269223e") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.vMrFaq1nPE
+ rm /tmp/tmp.9VPsDi0sm0 /tmp/tmp.vMrFaq1nPE
+ return 0
+ sleep 1
+ run_mongo_tls 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WpHboEgpw6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jLJkGUZ6EE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WpHboEgpw6
++ cat /tmp/tmp.jLJkGUZ6EE
++ rm /tmp/tmp.WpHboEgpw6 /tmp/tmp.jLJkGUZ6EE
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.j8FsPCujAN
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ch8V0m1TCa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j8FsPCujAN
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true
{"t":{"$date":"2026-04-09T16:13:02.926Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("794b725c-0b32-495c-8d14-4cff3980b364") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Ch8V0m1TCa
+ rm /tmp/tmp.j8FsPCujAN /tmp/tmp.Ch8V0m1TCa
+ return 0
+ sleep 5
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999 '' '' '' '' '' true
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=true
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-09T16:13:08+0000] running db.test.find() in myApp
+ [[ true == true ]]
+ mongo_command=run_mongo_tls
+ run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999 mongodb '' ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TXZx4ik4VI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lj8ocZsJxG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TXZx4ik4VI
++ cat /tmp/tmp.lj8ocZsJxG
++ rm /tmp/tmp.TXZx4ik4VI /tmp/tmp.lj8ocZsJxG
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tPIM6KvrFr
++ mktemp
+ local LAST_ERR=/tmp/tmp.lyNEV8crjn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tPIM6KvrFr
+ cat /tmp/tmp.lyNEV8crjn
+ rm /tmp/tmp.tPIM6KvrFr /tmp/tmp.lyNEV8crjn
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.u8fFA4mIJG/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999 '' '' '' '' '' true
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=true
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-09T16:13:10+0000] running db.test.find() in myApp
+ [[ true == true ]]
+ mongo_command=run_mongo_tls
+ run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2EZ63yptZN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pW1JmEEivW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2EZ63yptZN
++ cat /tmp/tmp.pW1JmEEivW
++ rm /tmp/tmp.2EZ63yptZN /tmp/tmp.pW1JmEEivW
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mq1zCDqNyG
++ mktemp
+ local LAST_ERR=/tmp/tmp.nilHWTcFFe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mq1zCDqNyG
+ cat /tmp/tmp.nilHWTcFFe
+ rm /tmp/tmp.mq1zCDqNyG /tmp/tmp.nilHWTcFFe
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.u8fFA4mIJG/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999 '' '' '' '' '' true
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=true
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-09T16:13:13+0000] running db.test.find() in myApp
+ [[ true == true ]]
+ mongo_command=run_mongo_tls
+ run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gZJfH9epRY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JatRgl9Ud9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gZJfH9epRY
++ cat /tmp/tmp.JatRgl9Ud9
++ rm /tmp/tmp.gZJfH9epRY /tmp/tmp.JatRgl9Ud9
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bB0a4vtHsu
++ mktemp
+ local LAST_ERR=/tmp/tmp.BV5bWREWjE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bB0a4vtHsu
+ cat /tmp/tmp.BV5bWREWjE
+ rm /tmp/tmp.bB0a4vtHsu /tmp/tmp.BV5bWREWjE
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.u8fFA4mIJG/find
+ desc 'Running snapshot backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Running snapshot backup
-----------------------------------------------------------------------------------

+ backup_name=backup-snapshot-vault
+ run_snapshot_backup backup-snapshot-vault
+ local backup_name=backup-snapshot-vault
+ log 'running snapshot backup backup-snapshot-vault'
+ set +o xtrace
[2026-04-09T16:13:16+0000] running snapshot backup backup-snapshot-vault
+ yq eval '.metadata.name = "backup-snapshot-vault" | .spec.volumeSnapshotClass = "gke-snapshot-class"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ooNVVdA7t0
++ mktemp
+ local LAST_ERR=/tmp/tmp.BsJq4MsZmK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ooNVVdA7t0
perconaservermongodbbackup.psmdb.percona.com/backup-snapshot-vault created
+ cat /tmp/tmp.BsJq4MsZmK
+ rm /tmp/tmp.ooNVVdA7t0 /tmp/tmp.BsJq4MsZmK
+ return 0
+ wait_backup backup-snapshot-vault
+ local backup_name=backup-snapshot-vault
+ local target_state=ready
+ set +o xtrace
waiting for backup-snapshot-vault to reach ready state......................OK
+ desc 'Drop collection and restore from snapshot'
+ set +o xtrace


-----------------------------------------------------------------------------------
Drop collection and restore from snapshot
-----------------------------------------------------------------------------------

+ run_mongo_tls 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WzzhQNIk7b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G0cNyDAznS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WzzhQNIk7b
++ cat /tmp/tmp.G0cNyDAznS
++ rm /tmp/tmp.WzzhQNIk7b /tmp/tmp.G0cNyDAznS
++ return 0
+ local client_container=psmdb-client-5649fbb65f-rw49f
+ local mongo_flag=
+ local port=27017
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ArHmWhlKMV
++ mktemp
+ local LAST_ERR=/tmp/tmp.ivdeQb4IfM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-5649fbb65f-rw49f -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ArHmWhlKMV
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true
{"t":{"$date":"2026-04-09T16:14:05.180Z"},"s":"I",  "c":"NETWORK",  "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"}
Implicit session: session { "id" : UUID("16dc6caf-35b0-4cda-aab9-8bc0ae62e38c") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.ivdeQb4IfM
+ rm /tmp/tmp.ArHmWhlKMV /tmp/tmp.ivdeQb4IfM
+ return 0
+ run_restore backup-snapshot-vault
+ local backup_name=backup-snapshot-vault
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/restore.yml
+ log 'running restore restore-backup-snapshot-vault'
+ set +o xtrace
[2026-04-09T16:14:05+0000] running restore restore-backup-snapshot-vault
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/demand-backup-snapshot-vault/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-snapshot-vault/'
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-snapshot-vault/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.WDG1pamelC
++ mktemp
+ local LAST_ERR=/tmp/tmp.elDL4VAz3m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WDG1pamelC
perconaservermongodbrestore.psmdb.percona.com/restore-backup-snapshot-vault created
+ cat /tmp/tmp.elDL4VAz3m
+ rm /tmp/tmp.WDG1pamelC /tmp/tmp.elDL4VAz3m
+ return 0
+ run_snapshot_recovery_check backup-snapshot-vault
+ local backup_name=backup-snapshot-vault
+ wait_restore backup-snapshot-vault some-name ready 0 3000
+ local backup_name=backup-snapshot-vault
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-snapshot-vault object to be created.OK
Waiting psmdb-restore/restore-backup-snapshot-vault to reach state "ready" ...2026-04-09T16:14:08.790Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "1ee77e09-5a5c-46f2-82f6-2663603fa3bd", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:08.791Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "1ee77e09-5a5c-46f2-82f6-2663603fa3bd", "previous": "", "current": "waiting"}
2026-04-09T16:14:09.429Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "bb08ab58-c626-4efd-830f-37745e545ec8", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:09.518Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "bb08ab58-c626-4efd-830f-37745e545ec8", "replset": "rs0"}
2026-04-09T16:14:09.552Z	INFO	Preparing statefulset for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "bb08ab58-c626-4efd-830f-37745e545ec8", "name": "some-name-rs0"}
2026-04-09T16:14:09.631Z	INFO	Updated statefulset	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "bb08ab58-c626-4efd-830f-37745e545ec8", "name": "some-name-rs0"}
2026-04-09T16:14:09.632Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "bb08ab58-c626-4efd-830f-37745e545ec8", "ready": false}
2026-04-09T16:14:14.633Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9fef06fe-4d1a-418c-9412-8294ab093913", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:14.683Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9fef06fe-4d1a-418c-9412-8294ab093913", "replset": "rs0"}
2026-04-09T16:14:14.684Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9fef06fe-4d1a-418c-9412-8294ab093913", "ready": false}
2026-04-09T16:14:19.685Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d11f3c8d-8529-4aad-a96f-262d2af48071", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:19.734Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d11f3c8d-8529-4aad-a96f-262d2af48071", "replset": "rs0"}
2026-04-09T16:14:19.735Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d11f3c8d-8529-4aad-a96f-262d2af48071", "ready": false}
2026-04-09T16:14:24.735Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ec8e120c-ffb1-437f-9390-18ba07e237de", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:24.784Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ec8e120c-ffb1-437f-9390-18ba07e237de", "replset": "rs0"}
2026-04-09T16:14:24.785Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ec8e120c-ffb1-437f-9390-18ba07e237de", "ready": false}
2026-04-09T16:14:29.786Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9b1b80c8-f61d-4ac2-b2c5-77251b1a0838", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:29.846Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9b1b80c8-f61d-4ac2-b2c5-77251b1a0838", "replset": "rs0"}
2026-04-09T16:14:29.846Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "9b1b80c8-f61d-4ac2-b2c5-77251b1a0838", "ready": false}
2026-04-09T16:14:34.847Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ca2e1586-2c9e-417b-ac94-233ad172b43e", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:34.908Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ca2e1586-2c9e-417b-ac94-233ad172b43e", "replset": "rs0"}
2026-04-09T16:14:34.908Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "ca2e1586-2c9e-417b-ac94-233ad172b43e", "ready": false}
2026-04-09T16:14:39.910Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "69cc0878-8300-4059-bdbf-8187d8c939f1", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:39.968Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "69cc0878-8300-4059-bdbf-8187d8c939f1", "replset": "rs0"}
2026-04-09T16:14:39.968Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "69cc0878-8300-4059-bdbf-8187d8c939f1", "ready": false}
2026-04-09T16:14:44.969Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "840af4e5-b123-43f8-b04e-8f599c5beca6", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:45.014Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "840af4e5-b123-43f8-b04e-8f599c5beca6", "replset": "rs0"}
2026-04-09T16:14:45.014Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "840af4e5-b123-43f8-b04e-8f599c5beca6", "ready": false}
2026-04-09T16:14:50.015Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d12a476d-4d59-40ca-b475-57946c84cb31", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:50.120Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d12a476d-4d59-40ca-b475-57946c84cb31", "replset": "rs0"}
2026-04-09T16:14:50.121Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "d12a476d-4d59-40ca-b475-57946c84cb31", "ready": false}
2026-04-09T16:14:55.122Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "45f704bd-1f08-408b-9b1f-e3f014722818", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:14:55.170Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "45f704bd-1f08-408b-9b1f-e3f014722818", "replset": "rs0"}
2026-04-09T16:14:55.170Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "45f704bd-1f08-408b-9b1f-e3f014722818", "ready": false}
2026-04-09T16:15:00.171Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "b460d47a-0698-497e-8bf9-8ca7091ba823", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:00.220Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "b460d47a-0698-497e-8bf9-8ca7091ba823", "replset": "rs0"}
2026-04-09T16:15:00.220Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "b460d47a-0698-497e-8bf9-8ca7091ba823", "ready": false}
2026-04-09T16:15:05.222Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "a825faa6-38ee-42d7-b087-950da97fe611", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:05.267Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "a825faa6-38ee-42d7-b087-950da97fe611", "replset": "rs0"}
2026-04-09T16:15:05.267Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "a825faa6-38ee-42d7-b087-950da97fe611", "ready": false}
2026-04-09T16:15:10.268Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "51b87e89-a102-43e0-b09b-2297182d6beb", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:10.321Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "51b87e89-a102-43e0-b09b-2297182d6beb", "replset": "rs0"}
2026-04-09T16:15:10.321Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "51b87e89-a102-43e0-b09b-2297182d6beb", "ready": false}
2026-04-09T16:15:15.322Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f7524caa-ba66-4c53-8e30-92ae470bb26f", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:15.373Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f7524caa-ba66-4c53-8e30-92ae470bb26f", "replset": "rs0"}
2026-04-09T16:15:15.373Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f7524caa-ba66-4c53-8e30-92ae470bb26f", "ready": false}
2026-04-09T16:15:20.374Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e3948716-de97-4517-99b2-a0f6654c84c4", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:20.422Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e3948716-de97-4517-99b2-a0f6654c84c4", "replset": "rs0"}
2026-04-09T16:15:20.422Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e3948716-de97-4517-99b2-a0f6654c84c4", "ready": false}
2026-04-09T16:15:25.423Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "db001d02-4daa-49a4-a872-dfeaaeb32f40", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:25.473Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "db001d02-4daa-49a4-a872-dfeaaeb32f40", "replset": "rs0"}
2026-04-09T16:15:25.473Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "db001d02-4daa-49a4-a872-dfeaaeb32f40", "ready": false}
2026-04-09T16:15:30.474Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "18a33a74-b7be-40dd-b9f7-13903c1ba40b", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:30.519Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "18a33a74-b7be-40dd-b9f7-13903c1ba40b", "replset": "rs0"}
2026-04-09T16:15:30.519Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "18a33a74-b7be-40dd-b9f7-13903c1ba40b", "ready": false}
2026-04-09T16:15:35.520Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "01b806db-6000-4441-b890-c59abf25d7e1", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:35.570Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "01b806db-6000-4441-b890-c59abf25d7e1", "replset": "rs0"}
2026-04-09T16:15:35.570Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "01b806db-6000-4441-b890-c59abf25d7e1", "pod": "some-name-rs0-2"}
2026-04-09T16:15:35.570Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "01b806db-6000-4441-b890-c59abf25d7e1", "ready": false}
2026-04-09T16:15:40.571Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "c40b077d-b478-4824-8163-e0e215184197", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:40.615Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "c40b077d-b478-4824-8163-e0e215184197", "replset": "rs0"}
2026-04-09T16:15:40.615Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "c40b077d-b478-4824-8163-e0e215184197", "ready": false}
2026-04-09T16:15:45.616Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "776be636-6297-4510-91a0-7a15f3d92269", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:45.661Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "776be636-6297-4510-91a0-7a15f3d92269", "replset": "rs0"}
2026-04-09T16:15:45.662Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "776be636-6297-4510-91a0-7a15f3d92269", "ready": false}
2026-04-09T16:15:50.663Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "934ade79-d634-445f-8461-2951e65e2eb4", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:50.708Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "934ade79-d634-445f-8461-2951e65e2eb4", "replset": "rs0"}
2026-04-09T16:15:50.708Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "934ade79-d634-445f-8461-2951e65e2eb4", "ready": false}
2026-04-09T16:15:55.709Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f2d5b35d-83b1-4df4-a065-aa77479841b1", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:15:55.753Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f2d5b35d-83b1-4df4-a065-aa77479841b1", "replset": "rs0"}
2026-04-09T16:15:55.753Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "f2d5b35d-83b1-4df4-a065-aa77479841b1", "ready": false}
2026-04-09T16:16:00.754Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "75b74d8d-82c8-4681-ad24-2643ac912391", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:16:00.798Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "75b74d8d-82c8-4681-ad24-2643ac912391", "replset": "rs0"}
2026-04-09T16:16:00.798Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "75b74d8d-82c8-4681-ad24-2643ac912391", "ready": false}
2026-04-09T16:16:05.799Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "63990316-0b5a-4b9b-b982-1f1c9da5aca6", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:16:10.162Z	INFO	Using vault security config for snapshot restore db-config	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "63990316-0b5a-4b9b-b982-1f1c9da5aca6", "replset": "rs0"}
2026-04-09T16:16:10.162Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "63990316-0b5a-4b9b-b982-1f1c9da5aca6", "ready": false}
2026-04-09T16:16:15.164Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e1b53135-cc8a-48fc-9904-75caa5b779a3", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-16999"}
2026-04-09T16:16:46.449Z	ERROR	failed to make restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e1b53135-cc8a-48fc-9904-75caa5b779a3", "restore": "restore-backup-snapshot-vault", "backup": "backup-snapshot-vault", "error": "update PBM config secret: new PBM connection: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017: create mongo connection: ping: connection() error occurred during connection handshake: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local: i/o timeout", "errorVerbose": "connection() error occurred during connection handshake: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local: i/o timeout\nping\ncreate mongo connection\ncreate PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:223\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).updatePBMConfigSecret\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:867\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:71\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nnew PBM connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).updatePBMConfigSecret\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:869\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:71\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdate PBM config secret\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:72\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T16:16:46.450Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e1b53135-cc8a-48fc-9904-75caa5b779a3", "previous": "waiting", "current": "error"}
2026-04-09T16:16:47.178Z	INFO	Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e1b53135-cc8a-48fc-9904-75caa5b779a3"}
2026-04-09T16:16:47.178Z	ERROR	Reconciler error	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"}, "namespace": "demand-backup-snapshot-vault-16999", "name": "restore-backup-snapshot-vault", "reconcileID": "e1b53135-cc8a-48fc-9904-75caa5b779a3", "error": "reconcile external snapshot restore: update PBM config secret: new PBM connection: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017: create mongo connection: ping: connection() error occurred during connection handshake: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local: i/o timeout", "errorVerbose": "connection() error occurred during connection handshake: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local: i/o timeout\nping\ncreate mongo connection\ncreate PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:223\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).updatePBMConfigSecret\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:867\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:71\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nnew PBM connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).updatePBMConfigSecret\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:869\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:71\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdate PBM config secret\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:72\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile external snapshot restore\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:268\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-16999"},"spec":{"backupName":"backup-snapshot-vault","clusterName":"some-name"}}
  creationTimestamp: "2026-04-09T16:14:07Z"
  generation: 1
  name: restore-backup-snapshot-vault
  namespace: demand-backup-snapshot-vault-16999
  resourceVersion: "1775751406549439005"
  uid: 72b48f1f-5f46-4c49-89aa-441875c9fe8e
spec:
  backupName: backup-snapshot-vault
  clusterName: some-name
status:
  error: 'update PBM config secret: new PBM connection: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017:
    create mongo connection: ping: connection() error occurred during connection handshake:
    dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:
    i/o timeout'
  state: error
Name:         restore-backup-snapshot-vault
Namespace:    demand-backup-snapshot-vault-16999
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2026-04-09T16:14:07Z
  Generation:          1
  Resource Version:    1775751406549439005
  UID:                 72b48f1f-5f46-4c49-89aa-441875c9fe8e
Spec:
  Backup Name:   backup-snapshot-vault
  Cluster Name:  some-name
Status:
  Error:  update PBM config secret: new PBM connection: create PBM connection to some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local:27017: create mongo connection: ping: connection() error occurred during connection handshake: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-16999.svc.cluster.local: i/o timeout
  State:  error
Events:   <none>
Restore object restore-backup-snapshot-vault is in error state after 3 minutes.
something went wrong with operator or kubernetes cluster