Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/logs/demand-backup-physical-sharded-gcp-native.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ '[' -n '' ']'
+ create_infra demand-backup-physical-sharded-gcp-native-7842
+ local ns=demand-backup-physical-sharded-gcp-native-7842
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.rhzqvDuBx9
++ mktemp
+ local LAST_ERR=/tmp/tmp.cBsviHCYmB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rhzqvDuBx9
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.cBsviHCYmB
+ rm /tmp/tmp.rhzqvDuBx9 /tmp/tmp.cBsviHCYmB
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-gcp-native-9901 backup-gcp-cs-sa-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa-sharded patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vtu95qOe44
++ mktemp
+ local LAST_ERR=/tmp/tmp.IgSHMo3EmF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vtu95qOe44
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.IgSHMo3EmF
+ rm /tmp/tmp.Vtu95qOe44 /tmp/tmp.IgSHMo3EmF
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.atfEttjco0
++ mktemp
+ local LAST_ERR=/tmp/tmp.QJDl4oBHFz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.atfEttjco0
+ cat /tmp/tmp.QJDl4oBHFz
+ rm /tmp/tmp.atfEttjco0 /tmp/tmp.QJDl4oBHFz
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.iYcq9wRmFC
++ mktemp
+ local LAST_ERR=/tmp/tmp.M26ubljvrC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iYcq9wRmFC
+ cat /tmp/tmp.M26ubljvrC
+ rm /tmp/tmp.iYcq9wRmFC /tmp/tmp.M26ubljvrC
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.AAYHIUmBDT
++ mktemp
+ local LAST_ERR=/tmp/tmp.u3VlRY8Ifk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AAYHIUmBDT
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.u3VlRY8Ifk
+ rm /tmp/tmp.AAYHIUmBDT /tmp/tmp.u3VlRY8Ifk
+ return 0
+ check_crd_for_deletion PR-2104-591a1b50
+ local git_tag=PR-2104-591a1b50
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2104-591a1b50/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eYrguNri4S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zywV1ewDJw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eYrguNri4S
++ cat /tmp/tmp.zywV1ewDJw
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eYrguNri4S
++ cat /tmp/tmp.zywV1ewDJw
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eYrguNri4S
++ cat /tmp/tmp.zywV1ewDJw
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.eYrguNri4S
++ cat /tmp/tmp.zywV1ewDJw
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.eYrguNri4S /tmp/tmp.zywV1ewDJw
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.158OSzDRvc
++ mktemp
+ local LAST_OUT=/tmp/tmp.woFJqJpI3m
++ mktemp
+ local LAST_ERR=/tmp/tmp.WUJMjso2hD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.8LbPxumTZU
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.158OSzDRvc
+ cat /tmp/tmp.WUJMjso2hD
+ rm /tmp/tmp.158OSzDRvc /tmp/tmp.WUJMjso2hD
+ return 0
namespace "demand-backup-physical-sharded-gcp-native-9901" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.woFJqJpI3m
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.8LbPxumTZU
+ rm /tmp/tmp.woFJqJpI3m /tmp/tmp.8LbPxumTZU
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.KQUvyLq4z4
++ mktemp
+ local LAST_ERR=/tmp/tmp.q6J87jtFXj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KQUvyLq4z4
+ cat /tmp/tmp.q6J87jtFXj
+ rm /tmp/tmp.KQUvyLq4z4 /tmp/tmp.q6J87jtFXj
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.sAsxXS2RCx
++ mktemp
+ local LAST_ERR=/tmp/tmp.OLqO4cehxo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sAsxXS2RCx
namespace/psmdb-operator created
+ cat /tmp/tmp.OLqO4cehxo
+ rm /tmp/tmp.sAsxXS2RCx /tmp/tmp.OLqO4cehxo
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.as4d4ehlIR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jCeFpkr9L4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.as4d4ehlIR
++ cat /tmp/tmp.jCeFpkr9L4
++ rm /tmp/tmp.as4d4ehlIR /tmp/tmp.jCeFpkr9L4
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.AMxSkefSjD
++ mktemp
+ local LAST_ERR=/tmp/tmp.gPSH5n62jE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AMxSkefSjD
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2" modified.
+ cat /tmp/tmp.gPSH5n62jE
+ rm /tmp/tmp.AMxSkefSjD /tmp/tmp.gPSH5n62jE
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2104-591a1b50'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2104-591a1b50
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RClIc4dxkj
++ mktemp
+ local LAST_ERR=/tmp/tmp.9nsKeGrj8L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RClIc4dxkj
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.9nsKeGrj8L
+ rm /tmp/tmp.RClIc4dxkj /tmp/tmp.9nsKeGrj8L
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EiVuNHooVy
++ mktemp
+ local LAST_ERR=/tmp/tmp.B7nmZVxE9L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EiVuNHooVy
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.B7nmZVxE9L
+ rm /tmp/tmp.EiVuNHooVy /tmp/tmp.B7nmZVxE9L
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2104-591a1b50") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZBxv4EekWC
++ mktemp
+ local LAST_ERR=/tmp/tmp.hz8uGBElEd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZBxv4EekWC
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.hz8uGBElEd
+ rm /tmp/tmp.ZBxv4EekWC /tmp/tmp.hz8uGBElEd
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ua3i3ZOq5q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8QJ8iHnZ12
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ua3i3ZOq5q
++ cat /tmp/tmp.8QJ8iHnZ12
++ rm /tmp/tmp.ua3i3ZOq5q /tmp/tmp.8QJ8iHnZ12
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-6cd4b44944-bdn85
+ local pod=percona-server-mongodb-operator-6cd4b44944-bdn85
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6cd4b44944-bdn85 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I0EXp6hl8M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.skFLG7PzXa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I0EXp6hl8M
++ cat /tmp/tmp.skFLG7PzXa
++ rm /tmp/tmp.I0EXp6hl8M /tmp/tmp.skFLG7PzXa
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6cd4b44944-bdn85
++ mktemp
+ local LAST_OUT=/tmp/tmp.IsIIZ5U22N
++ mktemp
+ local LAST_ERR=/tmp/tmp.HeikmkQkcd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6cd4b44944-bdn85
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IsIIZ5U22N
+ cat /tmp/tmp.HeikmkQkcd
+ rm /tmp/tmp.IsIIZ5U22N /tmp/tmp.HeikmkQkcd
+ return 0
2025-11-05T17:10:39.196Z	INFO	setup	Manager starting up	{"gitCommit": "591a1b50aa738bd0b38117b064923b562d47965f", "gitBranch": "PR-2104-591a1b50", "buildTime": "", "goVersion": "go1.25.3", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-physical-sharded-gcp-native-7842
+ local namespace=demand-backup-physical-sharded-gcp-native-7842
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ awk '{print$1}'
+ desc 'cleaned up old namespaces demand-backup-physical-sharded-gcp-native-7842'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-sharded-gcp-native-7842
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-physical-sharded-gcp-native-7842 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.0f2Mt5h7oj
+ local LAST_OUT=/tmp/tmp.JeTCj92SEz
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y1e13EnPoD
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.f7pu1BF8Z8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-physical-sharded-gcp-native-7842 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0f2Mt5h7oj
+ cat /tmp/tmp.Y1e13EnPoD
+ rm /tmp/tmp.0f2Mt5h7oj /tmp/tmp.Y1e13EnPoD
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JeTCj92SEz
+ cat /tmp/tmp.f7pu1BF8Z8
+ rm /tmp/tmp.JeTCj92SEz /tmp/tmp.f7pu1BF8Z8
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-gcp-native-7842
++ mktemp
+ local LAST_OUT=/tmp/tmp.t3QrqQUJFc
++ mktemp
+ local LAST_ERR=/tmp/tmp.xjvYZQAUl9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-sharded-gcp-native-7842
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t3QrqQUJFc
+ cat /tmp/tmp.xjvYZQAUl9
+ rm /tmp/tmp.t3QrqQUJFc /tmp/tmp.xjvYZQAUl9
+ return 0
+ desc 'create namespace demand-backup-physical-sharded-gcp-native-7842'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-physical-sharded-gcp-native-7842
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-physical-sharded-gcp-native-7842
++ mktemp
+ local LAST_OUT=/tmp/tmp.vW9TmFMSfi
++ mktemp
+ local LAST_ERR=/tmp/tmp.A4sUeA2Epa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-physical-sharded-gcp-native-7842
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vW9TmFMSfi
namespace/demand-backup-physical-sharded-gcp-native-7842 created
+ cat /tmp/tmp.A4sUeA2Epa
+ rm /tmp/tmp.vW9TmFMSfi /tmp/tmp.A4sUeA2Epa
+ return 0
+ set_kube_ctx demand-backup-physical-sharded-gcp-native-7842
+ local namespace=demand-backup-physical-sharded-gcp-native-7842
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mK7iWnCJPN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s6tjksNghc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mK7iWnCJPN
++ cat /tmp/tmp.s6tjksNghc
++ rm /tmp/tmp.mK7iWnCJPN /tmp/tmp.s6tjksNghc
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2 --namespace=demand-backup-physical-sharded-gcp-native-7842
++ mktemp
+ local LAST_OUT=/tmp/tmp.H77JLDYR2h
++ mktemp
+ local LAST_ERR=/tmp/tmp.mkRi6pjZlz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2 --namespace=demand-backup-physical-sharded-gcp-native-7842
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H77JLDYR2h
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2104-591a1b50-4-cluster2" modified.
+ cat /tmp/tmp.mkRi6pjZlz
+ rm /tmp/tmp.H77JLDYR2h /tmp/tmp.mkRi6pjZlz
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NvZaWemcoc
++ mktemp
+ local LAST_ERR=/tmp/tmp.SXIDxMkba2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NvZaWemcoc
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.SXIDxMkba2
+ rm /tmp/tmp.NvZaWemcoc /tmp/tmp.SXIDxMkba2
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ echo 'Creating PSMDB cluster'
Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Hif071WSgF
++ mktemp
+ local LAST_ERR=/tmp/tmp.ldn8K3jNky
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Hif071WSgF
secret/some-users created
+ cat /tmp/tmp.ldn8K3jNky
+ rm /tmp/tmp.Hif071WSgF /tmp/tmp.ldn8K3jNky
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2104-591a1b50"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.9bFd47l3TR
++ mktemp
+ local LAST_ERR=/tmp/tmp.gEEHwTWN6R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9bFd47l3TR
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.gEEHwTWN6R
+ rm /tmp/tmp.9bFd47l3TR /tmp/tmp.gEEHwTWN6R
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tICzxIzhaW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rkr4y0JQIh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tICzxIzhaW
deployment.apps/psmdb-client created
+ cat /tmp/tmp.Rkr4y0JQIh
+ rm /tmp/tmp.tICzxIzhaW /tmp/tmp.Rkr4y0JQIh
+ return 0
+ echo 'Check if all pods started'
Check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SinHW7vvIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M6y8VX7c9L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SinHW7vvIl
++ cat /tmp/tmp.M6y8VX7c9L
++ rm /tmp/tmp.SinHW7vvIl /tmp/tmp.M6y8VX7c9L
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8BgGVUFLbn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m8tWmSTI53
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8BgGVUFLbn
++ cat /tmp/tmp.m8tWmSTI53
++ rm /tmp/tmp.8BgGVUFLbn /tmp/tmp.m8tWmSTI53
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z5FZwxr648
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EP5ZEMdJgS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z5FZwxr648
++ cat /tmp/tmp.EP5ZEMdJgS
++ rm /tmp/tmp.Z5FZwxr648 /tmp/tmp.EP5ZEMdJgS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kt9dIp6mOO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M3mYpa0DQj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kt9dIp6mOO
++ cat /tmp/tmp.M3mYpa0DQj
++ rm /tmp/tmp.kt9dIp6mOO /tmp/tmp.M3mYpa0DQj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BHY7Ldnlc1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nsf9INb9sS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BHY7Ldnlc1
++ cat /tmp/tmp.Nsf9INb9sS
++ rm /tmp/tmp.BHY7Ldnlc1 /tmp/tmp.Nsf9INb9sS
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QKGtNgY8xa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6Bz9YOt9D6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QKGtNgY8xa
++ cat /tmp/tmp.6Bz9YOt9D6
++ rm /tmp/tmp.QKGtNgY8xa /tmp/tmp.6Bz9YOt9D6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JCnPz6Xwu5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uPUBKRcQbv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JCnPz6Xwu5
++ cat /tmp/tmp.uPUBKRcQbv
++ rm /tmp/tmp.JCnPz6Xwu5 /tmp/tmp.uPUBKRcQbv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9T6wXnK035
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wkpo8BWH0h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9T6wXnK035
++ cat /tmp/tmp.Wkpo8BWH0h
++ rm /tmp/tmp.9T6wXnK035 /tmp/tmp.Wkpo8BWH0h
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LFxoucJEMo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gJFeyBbc3Q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LFxoucJEMo
++ cat /tmp/tmp.gJFeyBbc3Q
++ rm /tmp/tmp.LFxoucJEMo /tmp/tmp.gJFeyBbc3Q
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rdPt9vGqw5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KOgq5otqzl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rdPt9vGqw5
++ cat /tmp/tmp.KOgq5otqzl
++ rm /tmp/tmp.rdPt9vGqw5 /tmp/tmp.KOgq5otqzl
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q6u8VcntPc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9y24XLdCHB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q6u8VcntPc
++ cat /tmp/tmp.9y24XLdCHB
++ rm /tmp/tmp.q6u8VcntPc /tmp/tmp.9y24XLdCHB
++ return 0
+ lbEndpoint=34.28.131.8
+ '[' -z 34.28.131.8 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nHbfjMAPRC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VBdM8BjUFU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nHbfjMAPRC
++ cat /tmp/tmp.VBdM8BjUFU
++ rm /tmp/tmp.nHbfjMAPRC /tmp/tmp.VBdM8BjUFU
++ return 0
+ local client_container=psmdb-client-8f86f7874-gntg8
+ kubectl_bin exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YhvW38ZBbh
++ mktemp
+ local LAST_ERR=/tmp/tmp.coRC5zihba
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YhvW38ZBbh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2def9bd5-3c7f-4f8f-834e-b975d69a58f5") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.coRC5zihba
+ rm /tmp/tmp.YhvW38ZBbh /tmp/tmp.coRC5zihba
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K2me1RZf17
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jUEhJDVvq7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K2me1RZf17
++ cat /tmp/tmp.jUEhJDVvq7
++ rm /tmp/tmp.K2me1RZf17 /tmp/tmp.jUEhJDVvq7
++ return 0
+ local client_container=psmdb-client-8f86f7874-gntg8
+ kubectl_bin exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TGJf5R9Ts8
++ mktemp
+ local LAST_ERR=/tmp/tmp.3SLQDX1T4f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TGJf5R9Ts8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b85fc12c-a882-4bc1-b2df-20cec3a5e291") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3SLQDX1T4f
+ rm /tmp/tmp.TGJf5R9Ts8 /tmp/tmp.3SLQDX1T4f
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842 mongodb '' '' 27017
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QZzFSRYpmN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WnLygPu0Qc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QZzFSRYpmN
++ cat /tmp/tmp.WnLygPu0Qc
++ rm /tmp/tmp.QZzFSRYpmN /tmp/tmp.WnLygPu0Qc
++ return 0
+ local client_container=psmdb-client-8f86f7874-gntg8
+ kubectl_bin exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FLqQZPdcmB
++ mktemp
+ local LAST_ERR=/tmp/tmp.oRJw6afbRs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FLqQZPdcmB
+ cat /tmp/tmp.oRJw6afbRs
+ rm /tmp/tmp.FLqQZPdcmB /tmp/tmp.oRJw6afbRs
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/find-sharded.json /tmp/tmp.yhcwiAnili/find-sharded
+ echo 'Waiting 60 seconds for stable timestamp in wiredtiger'
Waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ echo 'Running GCP storage backups'
Running GCP storage backups
+ backup_name_gcp=backup-gcp-cs-sa-sharded
+ run_backup gcp-cs-sa backup-gcp-cs-sa-sharded physical
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa-sharded
+ local type=physical
+ desc 'run backup backup-gcp-cs-sa-sharded'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-sa-sharded
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-sa-sharded"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.oUUuOq88DH
++ mktemp
+ local LAST_ERR=/tmp/tmp.FsCetr7kPC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oUUuOq88DH
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa-sharded created
+ cat /tmp/tmp.FsCetr7kPC
+ rm /tmp/tmp.oUUuOq88DH /tmp/tmp.FsCetr7kPC
+ return 0
+ wait_backup backup-gcp-cs-sa-sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa-sharded to reach ready state.......................OK
+ check_backup_in_storage backup-gcp-cs-sa-sharded gcs rs0
+ local backup=backup-gcp-cs-sa-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sa-sharded
++ local backup_name=backup-gcp-cs-sa-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.5g3ZknYCWg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N5VltvkAdp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5g3ZknYCWg
++ cat /tmp/tmp.N5VltvkAdp
++ rm /tmp/tmp.5g3ZknYCWg /tmp/tmp.N5VltvkAdp
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-11-05T17:16:11+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0  9126    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AOCedOGmBFC68uptuYYUfwZWoUsVHAxzOLbeW46DWU4CFUT5DkN5WRwGRh5_i1axW6dFsH3D3GEuliY
expires: Wed, 05 Nov 2025 18:16:11 GMT
date: Wed, 05 Nov 2025 17:16:11 GMT
cache-control: public, max-age=3600
last-modified: Wed, 05 Nov 2025 17:15:57 GMT
etag: "b521473aa70e95ef509d39803a806abc"
x-goog-generation: 1762362957043982
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 9126
x-goog-hash: crc32c=ePFg2A==
x-goog-hash: md5=tSFHOqcOle9QnTmAOoBqvA==
x-goog-expiration: Thu, 06 Nov 2025 17:15:57 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 9126
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sa-sharded gcs cfg
+ local backup=backup-gcp-cs-sa-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sa-sharded
++ local backup_name=backup-gcp-cs-sa-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.gARITK80Da
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JfZkC26BXz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gARITK80Da
++ cat /tmp/tmp.JfZkC26BXz
++ rm /tmp/tmp.gARITK80Da /tmp/tmp.JfZkC26BXz
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-11-05T17:16:11+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-05T17:15:39Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 17484    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AOCedOGrr_VEUVCjUZRLRDM_XvWbhGqZYOisLJeU57Bsq5lHnSklQB-0b_m26GSz-q_uQC2MinR8JnY
expires: Wed, 05 Nov 2025 18:16:11 GMT
date: Wed, 05 Nov 2025 17:16:11 GMT
cache-control: public, max-age=3600
last-modified: Wed, 05 Nov 2025 17:16:03 GMT
etag: "746ec923b189e89644f1c8bc1f675e8d"
x-goog-generation: 1762362963772863
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 17484
x-goog-hash: crc32c=bHd+Xg==
x-goog-hash: md5=dG7JI7GJ6JZE8ci8H2dejQ==
x-goog-expiration: Thu, 06 Nov 2025 17:16:03 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 17484
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ echo 'Drop collection'
Drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.smamr9DJmN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zFeBOzfb4L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.smamr9DJmN
++ cat /tmp/tmp.zFeBOzfb4L
++ rm /tmp/tmp.smamr9DJmN /tmp/tmp.zFeBOzfb4L
++ return 0
+ local client_container=psmdb-client-8f86f7874-gntg8
+ kubectl_bin exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FSTrmOKj7D
++ mktemp
+ local LAST_ERR=/tmp/tmp.yy9yWIARGy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-gntg8 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FSTrmOKj7D
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("c47994b8-7cb0-4935-94db-85aff8b1bd6d") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.yy9yWIARGy
+ rm /tmp/tmp.FSTrmOKj7D /tmp/tmp.yy9yWIARGy
+ return 0
+ echo 'Check backup and restore -- gcp-cs-sa'
Check backup and restore -- gcp-cs-sa
+ run_restore backup-gcp-cs-sa-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.oBXxidPGPx
++ mktemp
+ local LAST_ERR=/tmp/tmp.y79i5E2NNz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oBXxidPGPx
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa-sharded created
+ cat /tmp/tmp.y79i5E2NNz
+ rm /tmp/tmp.oBXxidPGPx /tmp/tmp.y79i5E2NNz
+ return 0
+ run_recovery_check backup-gcp-cs-sa-sharded _restore_sharded sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ local compare_suffix=_restore_sharded
+ local is_sharded=sharded
+ local tls=
+ wait_restore backup-gcp-cs-sa-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sa-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.yhcwiAnili/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-gcp-native-7842", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.IAuX0qDLa4
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xb1Qaq2Hh0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IAuX0qDLa4
+ cat /tmp/tmp.Xb1Qaq2Hh0
+ rm /tmp/tmp.IAuX0qDLa4 /tmp/tmp.Xb1Qaq2Hh0
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.yhcwiAnili/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.yhcwiAnili/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.yhcwiAnili/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2104/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.yhcwiAnili/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-11-05T17:20:33+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-gcp-cs-sa-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sa-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa-sharded to reach state "ready" ..................................................

-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-0
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.aU3bYrbyAF
++ mktemp
+ local LAST_ERR=/tmp/tmp.GpjaKJ3uy2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aU3bYrbyAF
2025/11/05 17:20:17 [entrypoint] starting `pbm-agent`
2025-11-05T17:20:21.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-05T17:20:21.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-05T17:20:21.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-05T17:20:21.000+0000 I starting PITR routine
2025-11-05T17:20:21.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017
2025-11-05T17:20:21.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-05T17:20:21.000+0000 I listening for the commands
2025-11-05T17:20:32.000+0000 I got command restore [name: 2025-11-05T17:20:32.030565046Z, snapshot: 2025-11-05T17:15:39Z] <ts: 1762363232>, opid: 690b8760949460fbd7e42104
2025-11-05T17:20:32.000+0000 I got epoch {1762363232 18}
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] backup: 2025-11-05T17:15:39Z
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] recovery started
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] port: 27558
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] wait for cluster status
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.GpjaKJ3uy2
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.aU3bYrbyAF /tmp/tmp.GpjaKJ3uy2
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-1
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.virm3KvzGt
++ mktemp
+ local LAST_ERR=/tmp/tmp.AVSiPFEXEP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.virm3KvzGt
2025/11/05 17:19:39 [entrypoint] starting `pbm-agent`
2025-11-05T17:19:43.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-05T17:19:43.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-05T17:19:43.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-05T17:19:43.000+0000 I starting PITR routine
2025-11-05T17:19:43.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017
2025-11-05T17:19:43.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-05T17:19:43.000+0000 I listening for the commands
2025-11-05T17:20:32.000+0000 I got command restore [name: 2025-11-05T17:20:32.030565046Z, snapshot: 2025-11-05T17:15:39Z] <ts: 1762363232>, opid: 690b8760949460fbd7e42104
2025-11-05T17:20:32.000+0000 I got epoch {1762362998 1}
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] oplog slicer disabled
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] backup: 2025-11-05T17:15:39Z
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] recovery started
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] port: 27048
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] wait for cluster status
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.AVSiPFEXEP
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.virm3KvzGt /tmp/tmp.AVSiPFEXEP
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-2
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.aaMBVMnd80
++ mktemp
+ local LAST_ERR=/tmp/tmp.oEQvESxqmQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aaMBVMnd80
2025/11/05 17:19:03 [entrypoint] starting `pbm-agent`
2025-11-05T17:19:07.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-05T17:19:07.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-05T17:19:07.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-05T17:19:07.000+0000 I starting PITR routine
2025-11-05T17:19:07.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017
2025-11-05T17:19:07.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-05T17:19:07.000+0000 I listening for the commands
2025-11-05T17:20:32.000+0000 I got command restore [name: 2025-11-05T17:20:32.030565046Z, snapshot: 2025-11-05T17:15:39Z] <ts: 1762363232>, opid: 690b8760949460fbd7e42104
2025-11-05T17:20:32.000+0000 I got epoch {1762363232 34}
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] backup: 2025-11-05T17:15:39Z
2025-11-05T17:20:32.000+0000 I [restore/2025-11-05T17:20:32.030565046Z] recovery started
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] port: 27414
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:20:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] wait for cluster status
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:22:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:24:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:26:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:28:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:30:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:32:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:34:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:36:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:38:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:40:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:42:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:44:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:46:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:48:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:50:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:52:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:54:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:56:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T17:58:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:00:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:02:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:04:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:06:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:08:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-7842.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:32.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-05T18:10:33.000+0000 D [restore/2025-11-05T17:20:32.030565046Z] uploading ".pbm.restore/2025-11-05T17:20:32.030565046Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.oEQvESxqmQ
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.aaMBVMnd80 /tmp/tmp.oEQvESxqmQ
+ return 0
+ grep restore-backup-gcp-cs-sa-sharded
+ tail -100
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xol0y3A2dF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JVn7QRSeyV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xol0y3A2dF
++ cat /tmp/tmp.JVn7QRSeyV
++ rm /tmp/tmp.Xol0y3A2dF /tmp/tmp.JVn7QRSeyV
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6cd4b44944-bdn85
++ mktemp
+ local LAST_OUT=/tmp/tmp.jMl4mYjZKL
++ mktemp
+ local LAST_ERR=/tmp/tmp.deGXYLx0gm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6cd4b44944-bdn85
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jMl4mYjZKL
+ cat /tmp/tmp.deGXYLx0gm
+ rm /tmp/tmp.jMl4mYjZKL /tmp/tmp.deGXYLx0gm
+ return 0
2025-11-05T18:10:18.072Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "40dd7489-3c99-4a98-ae64-fe69e741a624", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:18.072Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "40dd7489-3c99-4a98-ae64-fe69e741a624", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:18.072Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "40dd7489-3c99-4a98-ae64-fe69e741a624", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:19.476Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "40dd7489-3c99-4a98-ae64-fe69e741a624", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:24.477Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:24.479Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "pod": "some-name-rs0-0"}
2025-11-05T18:10:24.479Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "pod": "some-name-rs0-2"}
2025-11-05T18:10:24.479Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "pod": "some-name-rs0-1"}
2025-11-05T18:10:24.479Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:24.480Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:24.480Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:25.895Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "eb9507ec-58fc-4125-9b77-04592292e9b1", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:30.896Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:30.897Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "pod": "some-name-rs0-0"}
2025-11-05T18:10:30.897Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "pod": "some-name-rs0-2"}
2025-11-05T18:10:30.897Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "pod": "some-name-rs0-1"}
2025-11-05T18:10:30.897Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:30.897Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:30.897Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:32.130Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "11bed0ad-ab79-4d48-a2d1-690442d40800", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:37.131Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:37.132Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "pod": "some-name-rs0-1"}
2025-11-05T18:10:37.132Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "pod": "some-name-rs0-0"}
2025-11-05T18:10:37.132Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "pod": "some-name-rs0-2"}
2025-11-05T18:10:37.132Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:37.132Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:37.132Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:38.604Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "47a80e9d-a400-46bd-802e-b86135cc7e4b", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:43.605Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:43.607Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "pod": "some-name-rs0-0"}
2025-11-05T18:10:43.607Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "pod": "some-name-rs0-2"}
2025-11-05T18:10:43.607Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "pod": "some-name-rs0-1"}
2025-11-05T18:10:43.607Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:43.607Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:43.607Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:44.995Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "006c750b-7689-4f11-8243-faf0ac06f8b2", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:49.996Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:49.997Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "pod": "some-name-rs0-0"}
2025-11-05T18:10:49.997Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "pod": "some-name-rs0-2"}
2025-11-05T18:10:49.997Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "pod": "some-name-rs0-1"}
2025-11-05T18:10:49.997Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:49.997Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:49.997Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:51.161Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "83518084-0a6b-4bd5-a94f-61bea3360448", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:10:56.162Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:10:56.164Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "pod": "some-name-rs0-0"}
2025-11-05T18:10:56.164Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "pod": "some-name-rs0-2"}
2025-11-05T18:10:56.164Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "pod": "some-name-rs0-1"}
2025-11-05T18:10:56.164Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:10:56.164Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:10:56.164Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:10:57.240Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "200c02ef-0594-4377-9fc0-83dc77f4d3d4", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:02.241Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:02.243Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "pod": "some-name-rs0-1"}
2025-11-05T18:11:02.243Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "pod": "some-name-rs0-0"}
2025-11-05T18:11:02.243Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "pod": "some-name-rs0-2"}
2025-11-05T18:11:02.243Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:02.243Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:02.247Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:03.677Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "bc15246b-d874-4021-a06c-be75b77e779f", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:08.677Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:08.679Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "pod": "some-name-rs0-0"}
2025-11-05T18:11:08.679Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "pod": "some-name-rs0-2"}
2025-11-05T18:11:08.679Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "pod": "some-name-rs0-1"}
2025-11-05T18:11:08.679Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:08.679Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:08.679Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:09.967Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "dafdc6e1-4d2d-4229-ad79-14004b966d71", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:14.968Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:14.969Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "pod": "some-name-rs0-1"}
2025-11-05T18:11:14.969Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "pod": "some-name-rs0-0"}
2025-11-05T18:11:14.969Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "pod": "some-name-rs0-2"}
2025-11-05T18:11:14.969Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:14.969Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:14.969Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:16.117Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f1597cc2-7e84-4667-b1da-5acede0cf1f3", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:21.119Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:21.120Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "pod": "some-name-rs0-0"}
2025-11-05T18:11:21.120Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "pod": "some-name-rs0-2"}
2025-11-05T18:11:21.120Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "pod": "some-name-rs0-1"}
2025-11-05T18:11:21.120Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:21.120Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:21.120Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:22.234Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8230e29a-c522-4f4c-9339-6730c8010128", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:27.234Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:27.235Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "pod": "some-name-rs0-0"}
2025-11-05T18:11:27.235Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "pod": "some-name-rs0-2"}
2025-11-05T18:11:27.235Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "pod": "some-name-rs0-1"}
2025-11-05T18:11:27.235Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:27.235Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:27.235Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:28.582Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "f2b717ea-3e1b-4cc8-af73-f3c590a00760", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-05T18:11:33.584Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-7842"}
2025-11-05T18:11:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "pod": "some-name-rs0-2"}
2025-11-05T18:11:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "pod": "some-name-rs0-1"}
2025-11-05T18:11:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "pod": "some-name-rs0-0"}
2025-11-05T18:11:33.585Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-05T18:11:33.585Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-05T18:11:33.585Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-05T17:20:32.030565046Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-05T18:11:34.658Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"}, "namespace": "demand-backup-physical-sharded-gcp-native-7842", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "7a4afd98-53cd-42b4-8f04-41357ac0b6a7", "status": {"type":"physical","opid":"","name":"2025-11-05T17:20:32.030565046Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
+ kubectl_bin get psmdb-restore restore-backup-gcp-cs-sa-sharded -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yrOBGrjnec
++ mktemp
+ local LAST_ERR=/tmp/tmp.B2SMt0dCBG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-restore restore-backup-gcp-cs-sa-sharded -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yrOBGrjnec
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-7842"},"spec":{"backupName":"backup-gcp-cs-sa-sharded","clusterName":"some-name"}}
  creationTimestamp: "2025-11-05T17:16:14Z"
  generation: 1
  name: restore-backup-gcp-cs-sa-sharded
  namespace: demand-backup-physical-sharded-gcp-native-7842
  resourceVersion: "1762363232083231014"
  uid: ff38c1a8-d958-49fe-b176-ed33bec9932e
spec:
  backupName: backup-gcp-cs-sa-sharded
  clusterName: some-name
status:
  pbmName: "2025-11-05T17:20:32.030565046Z"
  state: requested
+ cat /tmp/tmp.B2SMt0dCBG
+ rm /tmp/tmp.yrOBGrjnec /tmp/tmp.B2SMt0dCBG
+ return 0
+ kubectl_bin describe psmdb-restore restore-backup-gcp-cs-sa-sharded
++ mktemp
+ local LAST_OUT=/tmp/tmp.qX4jl8gLG7
++ mktemp
+ local LAST_ERR=/tmp/tmp.141hhi9xdU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl describe psmdb-restore restore-backup-gcp-cs-sa-sharded
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qX4jl8gLG7
Name:         restore-backup-gcp-cs-sa-sharded
Namespace:    demand-backup-physical-sharded-gcp-native-7842
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2025-11-05T17:16:14Z
  Generation:          1
  Resource Version:    1762363232083231014
  UID:                 ff38c1a8-d958-49fe-b176-ed33bec9932e
Spec:
  Backup Name:   backup-gcp-cs-sa-sharded
  Cluster Name:  some-name
Status:
  Pbm Name:  2025-11-05T17:20:32.030565046Z
  State:     requested
Events:      <none>
+ cat /tmp/tmp.141hhi9xdU
+ rm /tmp/tmp.qX4jl8gLG7 /tmp/tmp.141hhi9xdU
+ return 0
+ echo 'Restore object restore-backup-gcp-cs-sa-sharded is in requested state after 50 minutes.'
Restore object restore-backup-gcp-cs-sa-sharded is in requested state after 50 minutes.
+ echo something went wrong with operator or kubernetes cluster
something went wrong with operator or kubernetes cluster
+ exit 1