Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/demand-backup-snapshot-vault.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + vault_name=vault-service ++ deploy_volume_snapshot_class ++ local snapshot_class_file ++ local k8s_env +++ detect_k8s_env +++ local provider=unknown +++ kubectl_bin api-resources +++ grep -q openshift.io ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VcSBrwK16k ++++ mktemp +++ local LAST_ERR=/tmp/tmp.x8mDd0wmT9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl api-resources +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.VcSBrwK16k +++ cat /tmp/tmp.x8mDd0wmT9 +++ rm /tmp/tmp.VcSBrwK16k /tmp/tmp.x8mDd0wmT9 +++ return 0 +++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}' +++ grep -q eks.amazonaws.com +++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}' +++ grep -q cloud.google.com/gke +++ provider=gke +++ echo gke ++ k8s_env=gke ++ case "$k8s_env" in ++ snapshot_class_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/gke-volume-snapshot-class.yaml ++ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/gke-volume-snapshot-class.yaml +++ mktemp ++ local LAST_OUT=/tmp/tmp.MyxfHHKQrY +++ mktemp ++ local LAST_ERR=/tmp/tmp.JdYvo6fCat ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/gke-volume-snapshot-class.yaml ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MyxfHHKQrY volumesnapshotclass.snapshot.storage.k8s.io/gke-snapshot-class created ++ cat /tmp/tmp.JdYvo6fCat ++ rm /tmp/tmp.MyxfHHKQrY /tmp/tmp.JdYvo6fCat ++ return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/gke-volume-snapshot-class.yaml + VOLUME_SNAPSHOT_CLASS=gke-snapshot-class + cluster=some-name + create_infra demand-backup-snapshot-vault-11354 + local ns=demand-backup-snapshot-vault-11354 + echo demand-backup-snapshot-vault-11354 + [[ 0 == 1 ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.A415GjN9yl ++ mktemp + local LAST_OUT=/tmp/tmp.tbf5EaYnyO ++ mktemp + local LAST_ERR=/tmp/tmp.tHzo6EogTW + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.mlOD9sQum0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A415GjN9yl + cat /tmp/tmp.tHzo6EogTW + rm /tmp/tmp.A415GjN9yl /tmp/tmp.tHzo6EogTW + return 0 namespace "demand-backup-incremental-sharded-minio-13424" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tbf5EaYnyO namespace "psmdb-operator" deleted + cat /tmp/tmp.mlOD9sQum0 + rm /tmp/tmp.tbf5EaYnyO /tmp/tmp.mlOD9sQum0 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dBZk8hKKuE ++ mktemp + local LAST_ERR=/tmp/tmp.ilI29wDuhb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dBZk8hKKuE + cat /tmp/tmp.ilI29wDuhb + rm /tmp/tmp.dBZk8hKKuE /tmp/tmp.ilI29wDuhb + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.qsm1RKRrbD ++ mktemp + local LAST_ERR=/tmp/tmp.A1Ie8N9uRq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qsm1RKRrbD namespace/psmdb-operator created + cat /tmp/tmp.A1Ie8N9uRq + rm /tmp/tmp.qsm1RKRrbD /tmp/tmp.A1Ie8N9uRq + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.UKfxiJQYz0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.X88ALbLyeN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UKfxiJQYz0 ++ cat /tmp/tmp.X88ALbLyeN ++ rm /tmp/tmp.UKfxiJQYz0 /tmp/tmp.X88ALbLyeN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.kD0nz9BcNN ++ mktemp + local LAST_ERR=/tmp/tmp.ONZcfgd6hU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kD0nz9BcNN Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15" modified. + cat /tmp/tmp.ONZcfgd6hU + rm /tmp/tmp.kD0nz9BcNN /tmp/tmp.ONZcfgd6hU + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mjKh4uY0wY ++ mktemp + local LAST_ERR=/tmp/tmp.6Douax8YPx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mjKh4uY0wY customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.6Douax8YPx + rm /tmp/tmp.mjKh4uY0wY /tmp/tmp.6Douax8YPx + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.A6JgpMCJ6U ++ mktemp + local LAST_ERR=/tmp/tmp.PtkjvgojOE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A6JgpMCJ6U clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged + cat /tmp/tmp.PtkjvgojOE + rm /tmp/tmp.A6JgpMCJ6U /tmp/tmp.PtkjvgojOE + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.p3mmu9CtFo ++ mktemp + local LAST_ERR=/tmp/tmp.oykx2N6Hih + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p3mmu9CtFo deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.oykx2N6Hih + rm /tmp/tmp.p3mmu9CtFo /tmp/tmp.oykx2N6Hih + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Te6LOh5e15 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gESXz95Vlm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Te6LOh5e15 ++ cat /tmp/tmp.gESXz95Vlm ++ rm /tmp/tmp.Te6LOh5e15 /tmp/tmp.gESXz95Vlm ++ return 0 + wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-4l956 + local pod=percona-server-mongodb-operator-859f4bc6d7-4l956 + set +o xtrace waiting for pod/percona-server-mongodb-operator-859f4bc6d7-4l956 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.0rdgN9brnt +++ mktemp ++ local LAST_ERR=/tmp/tmp.bjpoRglTFc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0rdgN9brnt ++ cat /tmp/tmp.bjpoRglTFc ++ rm /tmp/tmp.0rdgN9brnt /tmp/tmp.bjpoRglTFc ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-4l956 ++ mktemp + local LAST_OUT=/tmp/tmp.4vIWRVwXfM ++ mktemp + local LAST_ERR=/tmp/tmp.s4TS7VpEkN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-4l956 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4vIWRVwXfM + cat /tmp/tmp.s4TS7VpEkN + rm /tmp/tmp.4vIWRVwXfM /tmp/tmp.s4TS7VpEkN + return 0 2026-06-09T08:11:46.585Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-snapshot-vault-11354 + local namespace=demand-backup-snapshot-vault-11354 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + xargs kubectl delete ns + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-snapshot-vault-11354' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-snapshot-vault-11354 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-snapshot-vault-11354 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.6RDhnnxgRk ++ mktemp + local LAST_OUT=/tmp/tmp.IlPEWqKKq5 ++ mktemp + local LAST_ERR=/tmp/tmp.wQDbaFQ3DQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.wLGlPc8uLN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-snapshot-vault-11354 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6RDhnnxgRk + cat /tmp/tmp.wQDbaFQ3DQ + rm /tmp/tmp.6RDhnnxgRk /tmp/tmp.wQDbaFQ3DQ + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IlPEWqKKq5 + cat /tmp/tmp.wLGlPc8uLN + rm /tmp/tmp.IlPEWqKKq5 /tmp/tmp.wLGlPc8uLN + return 0 + kubectl_bin wait --for=delete namespace demand-backup-snapshot-vault-11354 ++ mktemp + local LAST_OUT=/tmp/tmp.kKYPji2zWq ++ mktemp + local LAST_ERR=/tmp/tmp.gbvdsZHdig + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-snapshot-vault-11354 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kKYPji2zWq + cat /tmp/tmp.gbvdsZHdig + rm /tmp/tmp.kKYPji2zWq /tmp/tmp.gbvdsZHdig + return 0 + desc 'create namespace demand-backup-snapshot-vault-11354' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-snapshot-vault-11354 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-snapshot-vault-11354 ++ mktemp + local LAST_OUT=/tmp/tmp.9zzV7lH10c ++ mktemp + local LAST_ERR=/tmp/tmp.VuchAkZCJB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-snapshot-vault-11354 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9zzV7lH10c namespace/demand-backup-snapshot-vault-11354 created + cat /tmp/tmp.VuchAkZCJB + rm /tmp/tmp.9zzV7lH10c /tmp/tmp.VuchAkZCJB + return 0 + set_kube_ctx demand-backup-snapshot-vault-11354 + local namespace=demand-backup-snapshot-vault-11354 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.rVnHqVl9zr +++ mktemp ++ local LAST_ERR=/tmp/tmp.owpxsebXVq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rVnHqVl9zr ++ cat /tmp/tmp.owpxsebXVq ++ rm /tmp/tmp.rVnHqVl9zr /tmp/tmp.owpxsebXVq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15 --namespace=demand-backup-snapshot-vault-11354 ++ mktemp + local LAST_OUT=/tmp/tmp.7VNtwCiXQP ++ mktemp + local LAST_ERR=/tmp/tmp.He1ItGsbcH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15 --namespace=demand-backup-snapshot-vault-11354 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7VNtwCiXQP Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster15" modified. + cat /tmp/tmp.He1ItGsbcH + rm /tmp/tmp.7VNtwCiXQP /tmp/tmp.He1ItGsbcH + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Tue Jun 9 08:12:25 2026 NAMESPACE: demand-backup-snapshot-vault-11354 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-snapshot-vault-11354.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-snapshot-vault-11354 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-snapshot-vault-11354 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-snapshot-vault-11354 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-snapshot-vault-11354 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CfuEii5xWX +++ mktemp ++ local LAST_ERR=/tmp/tmp.iEyXaVOJ0q ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CfuEii5xWX ++ cat /tmp/tmp.iEyXaVOJ0q ++ rm /tmp/tmp.CfuEii5xWX /tmp/tmp.iEyXaVOJ0q ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-jpzl8 + wait_pod minio-service-6d5f646cdc-jpzl8 + local pod=minio-service-6d5f646cdc-jpzl8 + set +o xtrace waiting for pod/minio-service-6d5f646cdc-jpzl8 to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-snapshot-vault-11354.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.HV3BUq3Lcj ++ mktemp + local LAST_ERR=/tmp/tmp.FpFYLxnOaK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HV3BUq3Lcj make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-snapshot-vault-11354 namespace + cat /tmp/tmp.FpFYLxnOaK All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.HV3BUq3Lcj /tmp/tmp.FpFYLxnOaK + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.r1ZDvAaPoC ++ mktemp + local LAST_ERR=/tmp/tmp.qW1iBm303d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.r1ZDvAaPoC secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.qW1iBm303d + rm /tmp/tmp.r1ZDvAaPoC /tmp/tmp.qW1iBm303d + return 0 + desc 'Setting up Vault' + set +o xtrace ----------------------------------------------------------------------------------- Setting up Vault ----------------------------------------------------------------------------------- + setup_vault + local sa_namespace=demand-backup-snapshot-vault-11354 + '[' -n psmdb-operator ']' + sa_namespace=psmdb-operator + deploy_vault vault-service --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true + local name=vault-service + [[ 7 -gt 0 ]] + shift + desc 'install Vault vault-service' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service ----------------------------------------------------------------------------------- + helm uninstall vault-service Error: uninstall: Release not loaded: vault-service: release: not found + : + helm repo remove hashicorp "hashicorp" has been removed from your repositories + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" has been added to your repositories + destroy_vault vault-service + local name=vault-service + local vault_ns ++ helm list --all-namespaces --filter vault-service ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + vault_ns= + desc 'destroy vault' + set +o xtrace ----------------------------------------------------------------------------------- destroy vault ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep vault ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get clusterrolebinding -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : ++ kubectl get mutatingwebhookconfiguration -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}' + timeout 30 kubectl delete mutatingwebhookconfiguration error: resource(s) were provided, but no name was specified + : + [[ -n '' ]] + retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true + local max=10 + local delay=60 + shift 2 + local n=1 + helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false --set global.enabled=true --set global.tlsDisable=true --set server.standalone.enabled=true NAME: vault-service LAST DEPLOYED: Tue Jun 9 08:13:41 2026 NAMESPACE: demand-backup-snapshot-vault-11354 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service. To learn more about the release, try: $ helm status vault-service $ helm get manifest vault-service + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running Running + sleep 5 + kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.q2uL3yrjqD ++ mktemp + local LAST_ERR=/tmp/tmp.GITZFNMU7O + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q2uL3yrjqD + cat /tmp/tmp.GITZFNMU7O + rm /tmp/tmp.q2uL3yrjqD /tmp/tmp.GITZFNMU7O + return 0 + local unsealKey ++ jq -r '.unseal_keys_b64[]' + unsealKey=zzXhxodtGWqrbnpJjRF+S9EJa2XayW6J8Nx7jGso0Z4= + local token ++ jq -r .root_token + token=hvs.FY2W1AdRmfDobFov2z6dMR9I + kubectl_bin exec pod/vault-service-0 -- vault operator unseal zzXhxodtGWqrbnpJjRF+S9EJa2XayW6J8Nx7jGso0Z4= ++ mktemp + local LAST_OUT=/tmp/tmp.E0s3a5qO0T ++ mktemp + local LAST_ERR=/tmp/tmp.vtSV3JoZzz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec pod/vault-service-0 -- vault operator unseal zzXhxodtGWqrbnpJjRF+S9EJa2XayW6J8Nx7jGso0Z4= + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E0s3a5qO0T Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 2.0.2 Build Date 2026-06-04T13:18:11Z Storage Type file Cluster Name vault-cluster-87b9c116 Cluster ID 6ccbaa18-ffb7-b48f-5874-e305b5555308 HA Enabled false + cat /tmp/tmp.vtSV3JoZzz + rm /tmp/tmp.E0s3a5qO0T /tmp/tmp.vtSV3JoZzz + return 0 + kubectl_bin exec -it pod/vault-service-0 -- sh ++ mktemp + local LAST_OUT=/tmp/tmp.ektwgDxOKa ++ mktemp + local LAST_ERR=/tmp/tmp.i0n98MktaI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec -it pod/vault-service-0 -- sh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ektwgDxOKa Success! You are now authenticated. The token information displayed below is already stored in the token helper. You do NOT need to run "vault login" again. Future Vault requests will automatically use this token. Key Value --- ----- token hvs.FY2W1AdRmfDobFov2z6dMR9I token_accessor rZKxrmDNb4EqtwygNZgKQ9lQ token_duration ∞ token_renewable false token_policies ["root"] identity_policies [] policies ["root"] Success! Enabled the kv-v2 secrets engine at: secret/ + cat /tmp/tmp.i0n98MktaI Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.ektwgDxOKa /tmp/tmp.i0n98MktaI + return 0 + kubectl_bin create secret generic vault-secret --from-literal=token=hvs.FY2W1AdRmfDobFov2z6dMR9I ++ mktemp + local LAST_OUT=/tmp/tmp.tX1mhkqI7C ++ mktemp + local LAST_ERR=/tmp/tmp.eQbCZKLwaV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic vault-secret --from-literal=token=hvs.FY2W1AdRmfDobFov2z6dMR9I + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tX1mhkqI7C secret/vault-secret created + cat /tmp/tmp.eQbCZKLwaV + rm /tmp/tmp.tX1mhkqI7C /tmp/tmp.eQbCZKLwaV + return 0 + sleep 10 + wait_pod vault-service-0 + local pod=vault-service-0 + set +o xtrace waiting for pod/vault-service-0 to be ready.OK + sleep 20 + kubectl_bin exec vault-service-0 -- vault auth enable kubernetes ++ mktemp + local LAST_OUT=/tmp/tmp.yDS8oBZ97K ++ mktemp + local LAST_ERR=/tmp/tmp.Uc5l6sgwGE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec vault-service-0 -- vault auth enable kubernetes + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yDS8oBZ97K Success! Enabled kubernetes auth method at: kubernetes/ + cat /tmp/tmp.Uc5l6sgwGE + rm /tmp/tmp.yDS8oBZ97K /tmp/tmp.Uc5l6sgwGE + return 0 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/role-binding.yml + yq '.metadata.namespace="demand-backup-snapshot-vault-11354"' + yq '.subjects[0].namespace="psmdb-operator"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.1iGox5p4Bs ++ mktemp + local LAST_ERR=/tmp/tmp.Jl5dLPvsju + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1iGox5p4Bs rolebinding.rbac.authorization.k8s.io/vault-role-binding created + cat /tmp/tmp.Jl5dLPvsju + rm /tmp/tmp.1iGox5p4Bs /tmp/tmp.Jl5dLPvsju + return 0 ++ kubectl_bin exec vault-service-0 -- vault token create -policy=operator -format=json ++ jq -r .auth.client_token +++ mktemp ++ local LAST_OUT=/tmp/tmp.9AJ7CRNGs2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.evgDlDCOIh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec vault-service-0 -- vault token create -policy=operator -format=json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9AJ7CRNGs2 ++ cat /tmp/tmp.evgDlDCOIh ++ rm /tmp/tmp.9AJ7CRNGs2 /tmp/tmp.evgDlDCOIh ++ return 0 + token=hvs.CAESIFDDi7Gc5BNvrS_3r5nEgQIq-p89fFZXyEGma5QuMO8PGh4KHGh2cy4yRmlvbUFPQmhBNzJuMTdVNzUzcWlxQXY + kubectl_bin create secret generic vault-sync-secret --from-literal=token=hvs.CAESIFDDi7Gc5BNvrS_3r5nEgQIq-p89fFZXyEGma5QuMO8PGh4KHGh2cy4yRmlvbUFPQmhBNzJuMTdVNzUzcWlxQXY ++ mktemp + local LAST_OUT=/tmp/tmp.erVA5fty7S ++ mktemp + local LAST_ERR=/tmp/tmp.D1IZNUvJBm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic vault-sync-secret --from-literal=token=hvs.CAESIFDDi7Gc5BNvrS_3r5nEgQIq-p89fFZXyEGma5QuMO8PGh4KHGh2cy4yRmlvbUFPQmhBNzJuMTdVNzUzcWlxQXY + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.erVA5fty7S secret/vault-sync-secret created + cat /tmp/tmp.D1IZNUvJBm + rm /tmp/tmp.erVA5fty7S /tmp/tmp.D1IZNUvJBm + return 0 + kubectl_bin exec vault-service-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT' ++ mktemp + local LAST_OUT=/tmp/tmp.IetfDGc35L ++ mktemp + local LAST_ERR=/tmp/tmp.0Na33UrL9g + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec vault-service-0 -- sh -c 'vault write auth/kubernetes/config kubernetes_host=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IetfDGc35L Success! Data written to: auth/kubernetes/config + cat /tmp/tmp.0Na33UrL9g + rm /tmp/tmp.IetfDGc35L /tmp/tmp.0Na33UrL9g + return 0 + kubectl_bin exec vault-service-0 -- sh -c $'vault policy write operator - < deletion_time n/a destroyed false version 1 + cat /tmp/tmp.Z6YzNM2Y0b + rm /tmp/tmp.DWScYzN71e /tmp/tmp.Z6YzNM2Y0b + return 0 + rm -f /tmp/tmp.Qss6ON42hD /tmp/tmp.J58gba8TfY + vault_append MONGODB_BACKUP_PASSWORD backup123456# + local key=MONGODB_BACKUP_PASSWORD + local value=backup123456# + local tmp_json ++ mktemp + tmp_json=/tmp/tmp.WLU7xBILa3 + local new_tmp_json ++ mktemp + new_tmp_json=/tmp/tmp.kXWXgWN2BF + kubectl_bin exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-11354/some-name/users' + jq .data.data ++ mktemp + local LAST_OUT=/tmp/tmp.hV13ftoiSR ++ mktemp + local LAST_ERR=/tmp/tmp.xyHKnirWOh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec vault-service-0 -- sh -c 'vault kv get -format=json -mount=secret psmdb/operator/demand-backup-snapshot-vault-11354/some-name/users' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hV13ftoiSR + cat /tmp/tmp.xyHKnirWOh + rm /tmp/tmp.hV13ftoiSR /tmp/tmp.xyHKnirWOh + return 0 + '[' '!' -s /tmp/tmp.WLU7xBILa3 ']' + jq --arg key MONGODB_BACKUP_PASSWORD --arg value backup123456# '(. // {}) + {($key): $value}' /tmp/tmp.WLU7xBILa3 + kubectl_bin cp /tmp/tmp.kXWXgWN2BF vault-service-0:/tmp/data_new.json ++ mktemp + local LAST_OUT=/tmp/tmp.umF3Argczs ++ mktemp + local LAST_ERR=/tmp/tmp.R6m7AS30Zf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /tmp/tmp.kXWXgWN2BF vault-service-0:/tmp/data_new.json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.umF3Argczs + cat /tmp/tmp.R6m7AS30Zf + rm /tmp/tmp.umF3Argczs /tmp/tmp.R6m7AS30Zf + return 0 + kubectl_bin exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-11354/some-name/users @"/tmp/data_new.json"' ++ mktemp + local LAST_OUT=/tmp/tmp.yCwaZAMOwB ++ mktemp + local LAST_ERR=/tmp/tmp.KE5jVnwgUD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec vault-service-0 -- sh -c 'vault kv put -mount=secret psmdb/operator/demand-backup-snapshot-vault-11354/some-name/users @"/tmp/data_new.json"' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yCwaZAMOwB ================================ Secret Path ================================ secret/data/psmdb/operator/demand-backup-snapshot-vault-11354/some-name/users ======= Metadata ======= Key Value --- ----- created_time 2026-06-09T08:15:27.164492209Z custom_metadata deletion_time n/a destroyed false version 2 + cat /tmp/tmp.KE5jVnwgUD + rm /tmp/tmp.yCwaZAMOwB /tmp/tmp.KE5jVnwgUD + return 0 + rm -f /tmp/tmp.WLU7xBILa3 /tmp/tmp.kXWXgWN2BF + desc 'Deploying PSMDB cluster with Vault and snapshot backup' + set +o xtrace ----------------------------------------------------------------------------------- Deploying PSMDB cluster with Vault and snapshot backup ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.HMo9sAHdLP ++ mktemp + local LAST_ERR=/tmp/tmp.YreUuku9RU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HMo9sAHdLP secret/some-users created + cat /tmp/tmp.YreUuku9RU + rm /tmp/tmp.HMo9sAHdLP /tmp/tmp.YreUuku9RU + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.LUFSfDy0lw + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-snapshot-vault-11354/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.F2eIIPR2Kr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LUFSfDy0lw perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.F2eIIPR2Kr + rm /tmp/tmp.LUFSfDy0lw /tmp/tmp.F2eIIPR2Kr + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MqGHPA6qkL ++ mktemp + local LAST_ERR=/tmp/tmp.ExChC6osf6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MqGHPA6qkL deployment.apps/psmdb-client created + cat /tmp/tmp.ExChC6osf6 + rm /tmp/tmp.MqGHPA6qkL /tmp/tmp.ExChC6osf6 + return 0 + echo 'check if all pods started' check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready...............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8LBfLpknPJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.HDC4oRuvy2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8LBfLpknPJ ++ cat /tmp/tmp.HDC4oRuvy2 ++ rm /tmp/tmp.8LBfLpknPJ /tmp/tmp.HDC4oRuvy2 ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ptpWnXLkH4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RP03wE6D0h ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ptpWnXLkH4 ++ cat /tmp/tmp.RP03wE6D0h ++ rm /tmp/tmp.ptpWnXLkH4 /tmp/tmp.RP03wE6D0h ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aENwlSEldS +++ mktemp ++ local LAST_ERR=/tmp/tmp.c4ScZ5hpfl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aENwlSEldS ++ cat /tmp/tmp.c4ScZ5hpfl ++ rm /tmp/tmp.aENwlSEldS /tmp/tmp.c4ScZ5hpfl ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness................ + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MOPCujieus +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xsza5IKI9f ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MOPCujieus ++ cat /tmp/tmp.Xsza5IKI9f ++ rm /tmp/tmp.MOPCujieus /tmp/tmp.Xsza5IKI9f ++ return 0 + [[ ready == ready ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + desc 'Writing test data' + set +o xtrace ----------------------------------------------------------------------------------- Writing test data ----------------------------------------------------------------------------------- + run_mongo_tls 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-11354 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ORAhHGbEN7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SbP5xbPmVr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ORAhHGbEN7 ++ cat /tmp/tmp.SbP5xbPmVr ++ rm /tmp/tmp.ORAhHGbEN7 /tmp/tmp.SbP5xbPmVr ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.93KDff0ZYE ++ mktemp + local LAST_ERR=/tmp/tmp.1B0dchCQO8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.93KDff0ZYE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-06-09T08:19:14.330Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("5dcfa134-5a16-4d58-a9b5-754815478aae") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.1B0dchCQO8 + rm /tmp/tmp.93KDff0ZYE /tmp/tmp.1B0dchCQO8 + return 0 + sleep 1 + run_mongo_tls 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k2gRPKyywQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.4V6F3AJJf2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.k2gRPKyywQ ++ cat /tmp/tmp.4V6F3AJJf2 ++ rm /tmp/tmp.k2gRPKyywQ /tmp/tmp.4V6F3AJJf2 ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.OsU5vTpMMa ++ mktemp + local LAST_ERR=/tmp/tmp.ie8M6oy8zt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OsU5vTpMMa Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-06-09T08:19:18.253Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("fb4a94e3-5d66-4b6d-8996-ed453e8a1c58") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ie8M6oy8zt + rm /tmp/tmp.OsU5vTpMMa /tmp/tmp.ie8M6oy8zt + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T08:19:23+0000] running db.test.find() in myApp + [[ true == true ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OEF27BhNt3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zMHXJZYNIA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OEF27BhNt3 ++ cat /tmp/tmp.zMHXJZYNIA ++ rm /tmp/tmp.OEF27BhNt3 /tmp/tmp.zMHXJZYNIA ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.284LIZkTsf ++ mktemp + local LAST_ERR=/tmp/tmp.UIZCaKBwdV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.284LIZkTsf + cat /tmp/tmp.UIZCaKBwdV + rm /tmp/tmp.284LIZkTsf /tmp/tmp.UIZCaKBwdV + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.RmacVQxapT/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T08:19:25+0000] running db.test.find() in myApp + [[ true == true ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ea2Modezth +++ mktemp ++ local LAST_ERR=/tmp/tmp.K3711VYVTc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ea2Modezth ++ cat /tmp/tmp.K3711VYVTc ++ rm /tmp/tmp.Ea2Modezth /tmp/tmp.K3711VYVTc ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.UsKtFCzQ9b ++ mktemp + local LAST_ERR=/tmp/tmp.vnstFEgcpF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UsKtFCzQ9b + cat /tmp/tmp.vnstFEgcpF + rm /tmp/tmp.UsKtFCzQ9b /tmp/tmp.vnstFEgcpF + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.RmacVQxapT/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T08:19:28+0000] running db.test.find() in myApp + [[ true == true ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bAR3CvwTNU +++ mktemp ++ local LAST_ERR=/tmp/tmp.GppqV7pMHv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bAR3CvwTNU ++ cat /tmp/tmp.GppqV7pMHv ++ rm /tmp/tmp.bAR3CvwTNU /tmp/tmp.GppqV7pMHv ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.UEqjdNMLl8 ++ mktemp + local LAST_ERR=/tmp/tmp.essJ2VDGjK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UEqjdNMLl8 + cat /tmp/tmp.essJ2VDGjK + rm /tmp/tmp.UEqjdNMLl8 /tmp/tmp.essJ2VDGjK + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/compare/find.json /tmp/tmp.RmacVQxapT/find + desc 'Running snapshot backup' + set +o xtrace ----------------------------------------------------------------------------------- Running snapshot backup ----------------------------------------------------------------------------------- + backup_name=backup-snapshot-vault + run_snapshot_backup backup-snapshot-vault + local backup_name=backup-snapshot-vault + log 'running snapshot backup backup-snapshot-vault' + set +o xtrace [2026-06-09T08:19:31+0000] running snapshot backup backup-snapshot-vault + yq eval '.metadata.name = "backup-snapshot-vault" | .spec.volumeSnapshotClass = "gke-snapshot-class"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Xgh7sruViO ++ mktemp + local LAST_ERR=/tmp/tmp.pMaLmVK7xI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Xgh7sruViO perconaservermongodbbackup.psmdb.percona.com/backup-snapshot-vault created + cat /tmp/tmp.pMaLmVK7xI + rm /tmp/tmp.Xgh7sruViO /tmp/tmp.pMaLmVK7xI + return 0 + wait_backup backup-snapshot-vault + local backup_name=backup-snapshot-vault + local target_state=ready + set +o xtrace waiting for backup-snapshot-vault to reach ready state......................OK + desc 'Drop collection and restore from snapshot' + set +o xtrace ----------------------------------------------------------------------------------- Drop collection and restore from snapshot ----------------------------------------------------------------------------------- + run_mongo_tls 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Er9pcZQxt +++ mktemp ++ local LAST_ERR=/tmp/tmp.eLTdvA7SUe ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6Er9pcZQxt ++ cat /tmp/tmp.eLTdvA7SUe ++ rm /tmp/tmp.6Er9pcZQxt /tmp/tmp.eLTdvA7SUe ++ return 0 + local client_container=psmdb-client-5649fbb65f-ggp5c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.liq0EKLmiy ++ mktemp + local LAST_ERR=/tmp/tmp.RKM0NnwLqt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-ggp5c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.liq0EKLmiy Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-vault-11354.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-06-09T08:20:19.016Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("bd39b7f2-8111-4e40-a8cb-5c143fcfcfbe") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.RKM0NnwLqt + rm /tmp/tmp.liq0EKLmiy /tmp/tmp.RKM0NnwLqt + return 0 + run_restore backup-snapshot-vault + local backup_name=backup-snapshot-vault + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/restore.yml + log 'running restore restore-backup-snapshot-vault' + set +o xtrace [2026-06-09T08:20:19+0000] running restore restore-backup-snapshot-vault + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-snapshot-vault/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-snapshot-vault/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-snapshot-vault/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.fa9i9Sgi7v ++ mktemp + local LAST_ERR=/tmp/tmp.6bOTeOUJpp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fa9i9Sgi7v perconaservermongodbrestore.psmdb.percona.com/restore-backup-snapshot-vault created + cat /tmp/tmp.6bOTeOUJpp + rm /tmp/tmp.fa9i9Sgi7v /tmp/tmp.6bOTeOUJpp + return 0 + run_snapshot_recovery_check backup-snapshot-vault + local backup_name=backup-snapshot-vault + wait_restore backup-snapshot-vault some-name ready 0 3000 + local backup_name=backup-snapshot-vault + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-snapshot-vault object to be created.OK Waiting psmdb-restore/restore-backup-snapshot-vault to reach state "ready" ....2026-06-09T08:20:21.052Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "b88c37e2-c15f-412b-ab7f-5bdcbb111874", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:21.099Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "b88c37e2-c15f-412b-ab7f-5bdcbb111874", "previous": "", "current": "waiting"} 2026-06-09T08:20:21.680Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "843ed661-3659-4264-95a9-5a5f8246f62c", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:21.862Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "843ed661-3659-4264-95a9-5a5f8246f62c", "replset": "rs0"} 2026-06-09T08:20:21.890Z INFO Preparing statefulset for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "843ed661-3659-4264-95a9-5a5f8246f62c", "name": "some-name-rs0"} 2026-06-09T08:20:21.936Z INFO Updated statefulset {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "843ed661-3659-4264-95a9-5a5f8246f62c", "name": "some-name-rs0"} 2026-06-09T08:20:21.937Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "843ed661-3659-4264-95a9-5a5f8246f62c", "ready": false} 2026-06-09T08:20:26.938Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7871c407-e1ea-48db-9683-b0c7ac16ed8b", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:27.043Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7871c407-e1ea-48db-9683-b0c7ac16ed8b", "replset": "rs0"} 2026-06-09T08:20:27.043Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7871c407-e1ea-48db-9683-b0c7ac16ed8b", "ready": false} 2026-06-09T08:20:32.044Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "25f8c3c2-f252-4a36-9021-ab12effde369", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:32.131Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "25f8c3c2-f252-4a36-9021-ab12effde369", "replset": "rs0"} 2026-06-09T08:20:32.131Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "25f8c3c2-f252-4a36-9021-ab12effde369", "ready": false} 2026-06-09T08:20:37.132Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "956007d9-f346-4cbd-a33c-24aa398e897a", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:37.225Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "956007d9-f346-4cbd-a33c-24aa398e897a", "replset": "rs0"} 2026-06-09T08:20:37.225Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "956007d9-f346-4cbd-a33c-24aa398e897a", "ready": false} 2026-06-09T08:20:42.226Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "47644759-df5e-48d3-aa1a-ed5f9dc4d2d9", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:42.326Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "47644759-df5e-48d3-aa1a-ed5f9dc4d2d9", "replset": "rs0"} 2026-06-09T08:20:42.326Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "47644759-df5e-48d3-aa1a-ed5f9dc4d2d9", "ready": false} 2026-06-09T08:20:47.328Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "556fb1ff-7361-4260-ae0f-4f5afd77f70f", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:47.410Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "556fb1ff-7361-4260-ae0f-4f5afd77f70f", "replset": "rs0"} 2026-06-09T08:20:47.410Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "556fb1ff-7361-4260-ae0f-4f5afd77f70f", "ready": false} 2026-06-09T08:20:52.411Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "4acc09e5-1044-4e0b-9ba3-3e69661fea80", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:52.494Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "4acc09e5-1044-4e0b-9ba3-3e69661fea80", "replset": "rs0"} 2026-06-09T08:20:52.494Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "4acc09e5-1044-4e0b-9ba3-3e69661fea80", "ready": false} 2026-06-09T08:20:57.494Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "9982bd02-1622-427f-85f5-750ea8ff8e9f", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:20:57.581Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "9982bd02-1622-427f-85f5-750ea8ff8e9f", "replset": "rs0"} 2026-06-09T08:20:57.581Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "9982bd02-1622-427f-85f5-750ea8ff8e9f", "ready": false} 2026-06-09T08:21:02.582Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7edfcf6b-6687-462a-bbaa-916efc680005", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:02.672Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7edfcf6b-6687-462a-bbaa-916efc680005", "replset": "rs0"} 2026-06-09T08:21:02.673Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7edfcf6b-6687-462a-bbaa-916efc680005", "ready": false} 2026-06-09T08:21:07.674Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1fc502df-0861-4331-8fb9-4082ab44bda0", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:07.758Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1fc502df-0861-4331-8fb9-4082ab44bda0", "replset": "rs0"} 2026-06-09T08:21:07.759Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1fc502df-0861-4331-8fb9-4082ab44bda0", "ready": false} 2026-06-09T08:21:12.760Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "77e966d2-bef4-43fb-b489-8b85bbab5f6d", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:12.849Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "77e966d2-bef4-43fb-b489-8b85bbab5f6d", "replset": "rs0"} 2026-06-09T08:21:12.850Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "77e966d2-bef4-43fb-b489-8b85bbab5f6d", "ready": false} 2026-06-09T08:21:17.850Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "df1a7e36-6aa8-4325-8292-2c787dc94f7b", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:17.938Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "df1a7e36-6aa8-4325-8292-2c787dc94f7b", "replset": "rs0"} 2026-06-09T08:21:17.939Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "df1a7e36-6aa8-4325-8292-2c787dc94f7b", "ready": false} 2026-06-09T08:21:22.940Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1748d5a2-9e8b-478d-9502-cc119921d281", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:23.032Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1748d5a2-9e8b-478d-9502-cc119921d281", "replset": "rs0"} 2026-06-09T08:21:23.033Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "1748d5a2-9e8b-478d-9502-cc119921d281", "ready": false} 2026-06-09T08:21:28.033Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0c16dc9-8f2e-4e86-bf1b-e560edd2c796", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:28.124Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0c16dc9-8f2e-4e86-bf1b-e560edd2c796", "replset": "rs0"} 2026-06-09T08:21:28.124Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0c16dc9-8f2e-4e86-bf1b-e560edd2c796", "ready": false} 2026-06-09T08:21:33.126Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "fb5292f2-b264-4725-99fa-d243105d16ce", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:44.031Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "fb5292f2-b264-4725-99fa-d243105d16ce", "replset": "rs0"} 2026-06-09T08:21:44.032Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "fb5292f2-b264-4725-99fa-d243105d16ce", "ready": false} 2026-06-09T08:21:49.032Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "863bd663-0e0c-46b8-bec9-9024077afe6b", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:49.095Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "863bd663-0e0c-46b8-bec9-9024077afe6b", "replset": "rs0"} 2026-06-09T08:21:49.095Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "863bd663-0e0c-46b8-bec9-9024077afe6b", "ready": false} 2026-06-09T08:21:54.095Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "c69549ee-b597-4527-8cab-ea4db409523e", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:54.163Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "c69549ee-b597-4527-8cab-ea4db409523e", "replset": "rs0"} 2026-06-09T08:21:54.163Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "c69549ee-b597-4527-8cab-ea4db409523e", "ready": false} 2026-06-09T08:21:59.164Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "607f4a2f-ad97-43ab-936e-3e6e44c290f3", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:21:59.234Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "607f4a2f-ad97-43ab-936e-3e6e44c290f3", "replset": "rs0"} 2026-06-09T08:21:59.235Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "607f4a2f-ad97-43ab-936e-3e6e44c290f3", "ready": false} 2026-06-09T08:22:04.236Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "31d25f3c-44f8-4ae6-a3ac-676a0ca3ddc3", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:04.305Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "31d25f3c-44f8-4ae6-a3ac-676a0ca3ddc3", "replset": "rs0"} 2026-06-09T08:22:04.306Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "31d25f3c-44f8-4ae6-a3ac-676a0ca3ddc3", "ready": false} 2026-06-09T08:22:09.306Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "81fdb011-9ddb-4498-8e6d-6aa579c4f134", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:09.374Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "81fdb011-9ddb-4498-8e6d-6aa579c4f134", "replset": "rs0"} 2026-06-09T08:22:09.374Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "81fdb011-9ddb-4498-8e6d-6aa579c4f134", "ready": false} 2026-06-09T08:22:14.375Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e7e484f3-f3bb-48cc-aecd-749cac31fff9", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:14.448Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e7e484f3-f3bb-48cc-aecd-749cac31fff9", "replset": "rs0"} 2026-06-09T08:22:14.448Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e7e484f3-f3bb-48cc-aecd-749cac31fff9", "ready": false} 2026-06-09T08:22:19.449Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e58596cc-f37b-4f90-a345-bcbff465eaeb", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:19.516Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e58596cc-f37b-4f90-a345-bcbff465eaeb", "replset": "rs0"} 2026-06-09T08:22:19.516Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e58596cc-f37b-4f90-a345-bcbff465eaeb", "ready": false} 2026-06-09T08:22:24.516Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7840f4b2-025d-433e-8775-f61ee055c6a9", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:24.585Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7840f4b2-025d-433e-8775-f61ee055c6a9", "replset": "rs0"} 2026-06-09T08:22:24.585Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "7840f4b2-025d-433e-8775-f61ee055c6a9", "ready": false} 2026-06-09T08:22:29.586Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e27fc962-5d1f-4cb9-a283-943ff9151fbd", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:29.654Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e27fc962-5d1f-4cb9-a283-943ff9151fbd", "replset": "rs0"} 2026-06-09T08:22:29.655Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "e27fc962-5d1f-4cb9-a283-943ff9151fbd", "ready": false} 2026-06-09T08:22:34.655Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "a9decf56-8328-46cf-8165-19659c4bf68c", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:34.720Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "a9decf56-8328-46cf-8165-19659c4bf68c", "replset": "rs0"} 2026-06-09T08:22:34.720Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "a9decf56-8328-46cf-8165-19659c4bf68c", "ready": false} 2026-06-09T08:22:39.722Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "ad03a19d-df21-4c83-a0fa-55dbacfc3d8a", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:39.789Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "ad03a19d-df21-4c83-a0fa-55dbacfc3d8a", "replset": "rs0"} 2026-06-09T08:22:39.790Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "ad03a19d-df21-4c83-a0fa-55dbacfc3d8a", "ready": false} 2026-06-09T08:22:44.790Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0d9f368-1920-4098-974f-39a1bddba463", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:44.859Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0d9f368-1920-4098-974f-39a1bddba463", "replset": "rs0"} 2026-06-09T08:22:44.860Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "f0d9f368-1920-4098-974f-39a1bddba463", "ready": false} 2026-06-09T08:22:49.860Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "6bfad053-f9a7-47d7-84a1-2ec0af297468", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:49.933Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "6bfad053-f9a7-47d7-84a1-2ec0af297468", "replset": "rs0"} 2026-06-09T08:22:49.933Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "6bfad053-f9a7-47d7-84a1-2ec0af297468", "ready": false} 2026-06-09T08:22:54.934Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "cluster": "some-name", "namespace": "demand-backup-snapshot-vault-11354"} 2026-06-09T08:22:55.004Z INFO Using vault security config for snapshot restore db-config {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "replset": "rs0"} 2026-06-09T08:22:55.004Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "pod": "some-name-rs0-0"} 2026-06-09T08:22:55.004Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "pod": "some-name-rs0-1"} 2026-06-09T08:22:55.004Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "pod": "some-name-rs0-2"} 2026-06-09T08:22:55.004Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "sts": "some-name-rs0", "replset": "rs0"} 2026-06-09T08:22:55.004Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "command": ["/opt/percona/pbm", "restore", "--external", "--out", "json", "--yes"], "pod": "some-name-rs0-0"} 2026-06-09T08:23:25.146Z ERROR Restore failed to start {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "pod": "some-name-rs0-0", "stderr": "", "stdout": "{\"Error\":\"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }\"}\n", "error": "command terminated with exit code 1"} 2026-06-09T08:23:25.146Z ERROR failed to make restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "restore": "restore-backup-snapshot-vault", "backup": "backup-snapshot-vault", "error": "start restore stderr: stdout: {\"Error\":\"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }\"}\n: command terminated with exit code 1", "errorVerbose": "command terminated with exit code 1\nstart restore stderr: stdout: {\"Error\":\"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }\"}\n\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting.func2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:126\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:114\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-06-09T08:23:25.146Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "previous": "waiting", "current": "error"} 2026-06-09T08:23:25.730Z INFO Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474"} 2026-06-09T08:23:25.731Z ERROR Reconciler error {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"}, "namespace": "demand-backup-snapshot-vault-11354", "name": "restore-backup-snapshot-vault", "reconcileID": "d29509ef-a62a-46b4-b561-5aacc52a4474", "error": "reconcile external snapshot restore: start restore stderr: stdout: {\"Error\":\"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }\"}\n: command terminated with exit code 1", "errorVerbose": "command terminated with exit code 1\nstart restore stderr: stdout: {\"Error\":\"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }\"}\n\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting.func2\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:126\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileSnapshotWaiting\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:114\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcileExternalSnapshotRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/snapshots.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:279\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile external snapshot restore\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:281\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-snapshot-vault","namespace":"demand-backup-snapshot-vault-11354"},"spec":{"backupName":"backup-snapshot-vault","clusterName":"some-name"}} creationTimestamp: "2026-06-09T08:20:21Z" generation: 1 name: restore-backup-snapshot-vault namespace: demand-backup-snapshot-vault-11354 resourceVersion: "1780993405166671013" uid: 2f925034-09a7-4045-ab9c-6f926de42977 spec: backupName: backup-snapshot-vault clusterName: some-name status: error: |- start restore stderr: stdout: {"Error":"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }"} : command terminated with exit code 1 state: error Name: restore-backup-snapshot-vault Namespace: demand-backup-snapshot-vault-11354 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2026-06-09T08:20:21Z Generation: 1 Resource Version: 1780993405166671013 UID: 2f925034-09a7-4045-ab9c-6f926de42977 Spec: Backup Name: backup-snapshot-vault Cluster Name: some-name Status: Error: start restore stderr: stdout: {"Error":"connect to mongodb: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] }"} : command terminated with exit code 1 State: error Events: Restore object restore-backup-snapshot-vault is in error state after 4 minutes. something went wrong with operator or kubernetes cluster