Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/logs/demand-backup-physical-sharded-gcp-native.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ '[' -n '' ']'
+ create_infra demand-backup-physical-sharded-gcp-native-12994
+ local ns=demand-backup-physical-sharded-gcp-native-12994
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.dfF5f1dQBI
++ mktemp
+ local LAST_ERR=/tmp/tmp.x84eWZ8Wil
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dfF5f1dQBI
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.x84eWZ8Wil
+ rm /tmp/tmp.dfF5f1dQBI /tmp/tmp.x84eWZ8Wil
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-gcp-native-9537 backup-gcp-cs-sa-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-gcp-cs-sa-sharded" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.NEOuwutcPK
++ mktemp
+ local LAST_ERR=/tmp/tmp.hb3FYf2lmV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NEOuwutcPK
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.hb3FYf2lmV
+ rm /tmp/tmp.NEOuwutcPK /tmp/tmp.hb3FYf2lmV
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.prXnplWkQB
++ mktemp
+ local LAST_ERR=/tmp/tmp.HpNOko0W6F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.prXnplWkQB
+ cat /tmp/tmp.HpNOko0W6F
+ rm /tmp/tmp.prXnplWkQB /tmp/tmp.HpNOko0W6F
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.snYOt2D1Hn
++ mktemp
+ local LAST_ERR=/tmp/tmp.YlBryP6oBE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.snYOt2D1Hn
+ cat /tmp/tmp.YlBryP6oBE
+ rm /tmp/tmp.snYOt2D1Hn /tmp/tmp.YlBryP6oBE
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RCUf1mwewl
++ mktemp
+ local LAST_ERR=/tmp/tmp.1SwEqNccIz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RCUf1mwewl
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.1SwEqNccIz
+ rm /tmp/tmp.RCUf1mwewl /tmp/tmp.1SwEqNccIz
+ return 0
+ check_crd_for_deletion PR-1850-4eb33c10
+ local git_tag=PR-1850-4eb33c10
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1850-4eb33c10/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.STacLP3DJk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xTHtyRseAV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STacLP3DJk
++ cat /tmp/tmp.xTHtyRseAV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STacLP3DJk
++ cat /tmp/tmp.xTHtyRseAV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.STacLP3DJk
++ cat /tmp/tmp.xTHtyRseAV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.STacLP3DJk
++ cat /tmp/tmp.xTHtyRseAV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.STacLP3DJk /tmp/tmp.xTHtyRseAV
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
egrep: warning: egrep is obsolescent; using grep -E
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.518FXHq0x7
++ mktemp
+ local LAST_OUT=/tmp/tmp.gZrceft51Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gn7ocfL2v1
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.cASejfjV2u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.518FXHq0x7
+ cat /tmp/tmp.Gn7ocfL2v1
+ rm /tmp/tmp.518FXHq0x7 /tmp/tmp.Gn7ocfL2v1
+ return 0
namespace "demand-backup-physical-sharded-gcp-native-9537" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gZrceft51Z
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.cASejfjV2u
+ rm /tmp/tmp.gZrceft51Z /tmp/tmp.cASejfjV2u
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.2iMgxUZj4V
++ mktemp
+ local LAST_ERR=/tmp/tmp.S2nDkdHJd1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2iMgxUZj4V
+ cat /tmp/tmp.S2nDkdHJd1
+ rm /tmp/tmp.2iMgxUZj4V /tmp/tmp.S2nDkdHJd1
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.xkmAAYJwVE
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dv5JCCQnYy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xkmAAYJwVE
namespace/psmdb-operator created
+ cat /tmp/tmp.Dv5JCCQnYy
+ rm /tmp/tmp.xkmAAYJwVE /tmp/tmp.Dv5JCCQnYy
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nt5M9x6PEv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MQ7XimSMAs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nt5M9x6PEv
++ cat /tmp/tmp.MQ7XimSMAs
++ rm /tmp/tmp.nt5M9x6PEv /tmp/tmp.MQ7XimSMAs
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.LSMJjBuBKG
++ mktemp
+ local LAST_ERR=/tmp/tmp.zzuC9FaB5P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LSMJjBuBKG
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10" modified.
+ cat /tmp/tmp.zzuC9FaB5P
+ rm /tmp/tmp.LSMJjBuBKG /tmp/tmp.zzuC9FaB5P
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1850-4eb33c10'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1850-4eb33c10
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9DJ5uItQco
++ mktemp
+ local LAST_ERR=/tmp/tmp.1OFEeE0HUd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9DJ5uItQco
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.1OFEeE0HUd
+ rm /tmp/tmp.9DJ5uItQco /tmp/tmp.1OFEeE0HUd
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1G9qrLWbvA
++ mktemp
+ local LAST_ERR=/tmp/tmp.CVuyBmpt6N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1G9qrLWbvA
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.CVuyBmpt6N
+ rm /tmp/tmp.1G9qrLWbvA /tmp/tmp.CVuyBmpt6N
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1850-4eb33c10") |
+ kubectl_bin apply -n psmdb-operator -f -
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RDZayjzT81
++ mktemp
+ local LAST_ERR=/tmp/tmp.90eHD7Np3p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RDZayjzT81
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.90eHD7Np3p
+ rm /tmp/tmp.RDZayjzT81 /tmp/tmp.90eHD7Np3p
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jU3gwvwoY6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TD5dQzKNs6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jU3gwvwoY6
++ cat /tmp/tmp.TD5dQzKNs6
++ rm /tmp/tmp.jU3gwvwoY6 /tmp/tmp.TD5dQzKNs6
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-67bdb49f8b-kl79d
+ local pod=percona-server-mongodb-operator-67bdb49f8b-kl79d
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-67bdb49f8b-kl79d to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zkug8VtzP5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eJiqZrXWSj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zkug8VtzP5
++ cat /tmp/tmp.eJiqZrXWSj
++ rm /tmp/tmp.Zkug8VtzP5 /tmp/tmp.eJiqZrXWSj
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-67bdb49f8b-kl79d
++ mktemp
+ local LAST_OUT=/tmp/tmp.2jS8wLna4v
++ mktemp
+ local LAST_ERR=/tmp/tmp.XbcvS7t9x9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-67bdb49f8b-kl79d
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2jS8wLna4v
+ cat /tmp/tmp.XbcvS7t9x9
+ rm /tmp/tmp.2jS8wLna4v /tmp/tmp.XbcvS7t9x9
+ return 0
2025-11-10T07:31:46.997Z	INFO	setup	Manager starting up	{"gitCommit": "4eb33c104d0cbf3fd7123b6a71ad408498660a69", "gitBranch": "PR-1850-4eb33c10", "buildTime": "", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-physical-sharded-gcp-native-12994
+ local namespace=demand-backup-physical-sharded-gcp-native-12994
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-physical-sharded-gcp-native-12994'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-sharded-gcp-native-12994
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace demand-backup-physical-sharded-gcp-native-12994 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.qyRqNf8LKd
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_ERR=/tmp/tmp.M62s7TyzN5
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.Tn3ubEQwR4
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.FTKyaDaZqv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-physical-sharded-gcp-native-12994 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qyRqNf8LKd
+ cat /tmp/tmp.M62s7TyzN5
+ rm /tmp/tmp.qyRqNf8LKd /tmp/tmp.M62s7TyzN5
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Tn3ubEQwR4
+ cat /tmp/tmp.FTKyaDaZqv
+ rm /tmp/tmp.Tn3ubEQwR4 /tmp/tmp.FTKyaDaZqv
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-gcp-native-12994
++ mktemp
+ local LAST_OUT=/tmp/tmp.tMjymCB0zH
++ mktemp
+ local LAST_ERR=/tmp/tmp.MrHIhNVrQe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-sharded-gcp-native-12994
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tMjymCB0zH
+ cat /tmp/tmp.MrHIhNVrQe
+ rm /tmp/tmp.tMjymCB0zH /tmp/tmp.MrHIhNVrQe
+ return 0
+ desc 'create namespace demand-backup-physical-sharded-gcp-native-12994'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-physical-sharded-gcp-native-12994
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-physical-sharded-gcp-native-12994
++ mktemp
+ local LAST_OUT=/tmp/tmp.e4Jm4M269m
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z4TXHGJhim
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-physical-sharded-gcp-native-12994
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.e4Jm4M269m
namespace/demand-backup-physical-sharded-gcp-native-12994 created
+ cat /tmp/tmp.Z4TXHGJhim
+ rm /tmp/tmp.e4Jm4M269m /tmp/tmp.Z4TXHGJhim
+ return 0
+ set_kube_ctx demand-backup-physical-sharded-gcp-native-12994
+ local namespace=demand-backup-physical-sharded-gcp-native-12994
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hT8nCxOXit
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MYGDGYW1RY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hT8nCxOXit
++ cat /tmp/tmp.MYGDGYW1RY
++ rm /tmp/tmp.hT8nCxOXit /tmp/tmp.MYGDGYW1RY
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10 --namespace=demand-backup-physical-sharded-gcp-native-12994
++ mktemp
+ local LAST_OUT=/tmp/tmp.u75ta6HIsS
++ mktemp
+ local LAST_ERR=/tmp/tmp.84VBctM6Nz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10 --namespace=demand-backup-physical-sharded-gcp-native-12994
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u75ta6HIsS
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1850-4eb33c10-13-cluster10" modified.
+ cat /tmp/tmp.84VBctM6Nz
+ rm /tmp/tmp.u75ta6HIsS /tmp/tmp.84VBctM6Nz
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.C3uhp7VorV
++ mktemp
+ local LAST_ERR=/tmp/tmp.7YSQmasQRi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C3uhp7VorV
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.7YSQmasQRi
+ rm /tmp/tmp.C3uhp7VorV /tmp/tmp.7YSQmasQRi
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ echo 'Creating PSMDB cluster'
Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.02AbGKGwkj
++ mktemp
+ local LAST_ERR=/tmp/tmp.84B2a8U8BP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.02AbGKGwkj
secret/some-users created
+ cat /tmp/tmp.84B2a8U8BP
+ rm /tmp/tmp.02AbGKGwkj /tmp/tmp.84B2a8U8BP
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/some-name-sharded.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1850-4eb33c10"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.3DP8x2d3GC
++ mktemp
+ local LAST_ERR=/tmp/tmp.VTiS7rgoD6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3DP8x2d3GC
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.VTiS7rgoD6
+ rm /tmp/tmp.3DP8x2d3GC /tmp/tmp.VTiS7rgoD6
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.lRZbP0xiel
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZDvj7MX4gX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lRZbP0xiel
deployment.apps/psmdb-client created
+ cat /tmp/tmp.ZDvj7MX4gX
+ rm /tmp/tmp.lRZbP0xiel /tmp/tmp.ZDvj7MX4gX
+ return 0
+ echo 'Check if all pods started'
Check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LICXtwYmjr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vi8nbDy9p0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LICXtwYmjr
++ cat /tmp/tmp.vi8nbDy9p0
++ rm /tmp/tmp.LICXtwYmjr /tmp/tmp.vi8nbDy9p0
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ydTjwPTE3j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RhozdTFhqz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ydTjwPTE3j
++ cat /tmp/tmp.RhozdTFhqz
++ rm /tmp/tmp.ydTjwPTE3j /tmp/tmp.RhozdTFhqz
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.InRVrWiqqi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NXDTACLncR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.InRVrWiqqi
++ cat /tmp/tmp.NXDTACLncR
++ rm /tmp/tmp.InRVrWiqqi /tmp/tmp.NXDTACLncR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness............................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A1vczbTwme
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0rvgMKRmuD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A1vczbTwme
++ cat /tmp/tmp.0rvgMKRmuD
++ rm /tmp/tmp.A1vczbTwme /tmp/tmp.0rvgMKRmuD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jd0H5zDenb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6181FI7NYd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jd0H5zDenb
++ cat /tmp/tmp.6181FI7NYd
++ rm /tmp/tmp.jd0H5zDenb /tmp/tmp.6181FI7NYd
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2IzRXEdJwV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eA5g45XamM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2IzRXEdJwV
++ cat /tmp/tmp.eA5g45XamM
++ rm /tmp/tmp.2IzRXEdJwV /tmp/tmp.eA5g45XamM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d3LVTlXKZb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.my4EsffrxJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d3LVTlXKZb
++ cat /tmp/tmp.my4EsffrxJ
++ rm /tmp/tmp.d3LVTlXKZb /tmp/tmp.my4EsffrxJ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fp07jBBSnc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ECd5WFzCVW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fp07jBBSnc
++ cat /tmp/tmp.ECd5WFzCVW
++ rm /tmp/tmp.fp07jBBSnc /tmp/tmp.ECd5WFzCVW
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MgBVsOvZCN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kGKpmkJ6tg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MgBVsOvZCN
++ cat /tmp/tmp.kGKpmkJ6tg
++ rm /tmp/tmp.MgBVsOvZCN /tmp/tmp.kGKpmkJ6tg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NmcNC6m22K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OX2JF68UWO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NmcNC6m22K
++ cat /tmp/tmp.OX2JF68UWO
++ rm /tmp/tmp.NmcNC6m22K /tmp/tmp.OX2JF68UWO
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DQOBkSzFOu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kaWSgXAaPs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DQOBkSzFOu
++ cat /tmp/tmp.kaWSgXAaPs
++ rm /tmp/tmp.DQOBkSzFOu /tmp/tmp.kaWSgXAaPs
++ return 0
+ lbEndpoint=35.184.230.7
+ '[' -z 35.184.230.7 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OPMv0KfLfg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yNoz4fMLY4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OPMv0KfLfg
++ cat /tmp/tmp.yNoz4fMLY4
++ rm /tmp/tmp.OPMv0KfLfg /tmp/tmp.yNoz4fMLY4
++ return 0
+ local client_container=psmdb-client-8f86f7874-5s278
+ kubectl_bin exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.r2iA6qXGKg
++ mktemp
+ local LAST_ERR=/tmp/tmp.FzXFtWpTpz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r2iA6qXGKg
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("9f5ba9fe-37f9-4028-9d4b-743bed945723") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.FzXFtWpTpz
+ rm /tmp/tmp.r2iA6qXGKg /tmp/tmp.FzXFtWpTpz
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rj21ry1mHn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VidbZAD5BK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Rj21ry1mHn
++ cat /tmp/tmp.VidbZAD5BK
++ rm /tmp/tmp.Rj21ry1mHn /tmp/tmp.VidbZAD5BK
++ return 0
+ local client_container=psmdb-client-8f86f7874-5s278
+ kubectl_bin exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qm6NPjCWPB
++ mktemp
+ local LAST_ERR=/tmp/tmp.L386l5oq4o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qm6NPjCWPB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("774d74c1-6624-43a6-a2e1-c1a10a0bf2aa") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.L386l5oq4o
+ rm /tmp/tmp.qm6NPjCWPB /tmp/tmp.L386l5oq4o
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YJl7XGnq6V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yIf2rVfzdf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YJl7XGnq6V
++ cat /tmp/tmp.yIf2rVfzdf
++ rm /tmp/tmp.YJl7XGnq6V /tmp/tmp.yIf2rVfzdf
++ return 0
+ local client_container=psmdb-client-8f86f7874-5s278
+ kubectl_bin exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.f74Y13pDuV
++ mktemp
+ local LAST_ERR=/tmp/tmp.WPqLipaFLk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f74Y13pDuV
+ cat /tmp/tmp.WPqLipaFLk
+ rm /tmp/tmp.f74Y13pDuV /tmp/tmp.WPqLipaFLk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/find-sharded.json /tmp/tmp.UCrNrKEpwi/find-sharded
+ echo 'Waiting 60 seconds for stable timestamp in wiredtiger'
Waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ echo 'Running GCP storage backups'
Running GCP storage backups
+ backup_name_gcp=backup-gcp-cs-sa-sharded
+ run_backup gcp-cs-sa backup-gcp-cs-sa-sharded physical
+ local storage=gcp-cs-sa
+ local backup_name=backup-gcp-cs-sa-sharded
+ local type=physical
+ desc 'run backup backup-gcp-cs-sa-sharded'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs-sa-sharded
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs-sa-sharded"
			| .spec.storageName = "gcp-cs-sa"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/backup-gcp-cs-sa.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FQSo40BR5c
++ mktemp
+ local LAST_ERR=/tmp/tmp.tI1NfUkhNX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FQSo40BR5c
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa-sharded created
+ cat /tmp/tmp.tI1NfUkhNX
+ rm /tmp/tmp.FQSo40BR5c /tmp/tmp.tI1NfUkhNX
+ return 0
+ wait_backup backup-gcp-cs-sa-sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sa-sharded to reach ready state................OK
+ check_backup_in_storage backup-gcp-cs-sa-sharded gcs rs0
+ local backup=backup-gcp-cs-sa-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sa-sharded
++ local backup_name=backup-gcp-cs-sa-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|gs://||'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.FOFaywvw4l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BGkZyPO91o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FOFaywvw4l
++ cat /tmp/tmp.BGkZyPO91o
++ rm /tmp/tmp.FOFaywvw4l /tmp/tmp.BGkZyPO91o
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-11-10T07:37:31+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0  9126    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AOCedOG2xKz5sbMDtvHuxwu_GOojIHdapgdo1AAGNLEpVZlxPNUIdfhqVw4zocFLvPoGIQ-2fO_SBAE
expires: Mon, 10 Nov 2025 08:37:31 GMT
date: Mon, 10 Nov 2025 07:37:31 GMT
cache-control: public, max-age=3600
last-modified: Mon, 10 Nov 2025 07:37:15 GMT
etag: "37f0c317804e6a6f733b2df0f393d09c"
x-goog-generation: 1762760235882241
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 9126
x-goog-hash: crc32c=mdiTJw==
x-goog-hash: md5=N/DDF4BOam9zOy3w85PQnA==
x-goog-expiration: Tue, 11 Nov 2025 07:37:15 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 9126
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sa-sharded gcs cfg
+ local backup=backup-gcp-cs-sa-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sa-sharded
++ local backup_name=backup-gcp-cs-sa-sharded
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed -e 's/.json$//'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bchgZ4BOIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qJlP9FDzuW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sa-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bchgZ4BOIl
++ cat /tmp/tmp.qJlP9FDzuW
++ rm /tmp/tmp.bchgZ4BOIl /tmp/tmp.qJlP9FDzuW
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-11-10T07:37:31+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-11-10T07:37:02Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 17509    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AOCedOGLw52seF_h0xfDLENEozvQ7dQmwjtsdIMpxxSW1WmTQQWQZyibCzPJzygjc_I62u6dQADKsD4
expires: Mon, 10 Nov 2025 08:37:31 GMT
date: Mon, 10 Nov 2025 07:37:31 GMT
cache-control: public, max-age=3600
last-modified: Mon, 10 Nov 2025 07:37:21 GMT
etag: "c58b238fa0d69a0dd6873f9661d7d73b"
x-goog-generation: 1762760241619806
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 17509
x-goog-hash: crc32c=amdFyg==
x-goog-hash: md5=xYsjj6DWmg3Whz+WYdfXOw==
x-goog-expiration: Tue, 11 Nov 2025 07:37:21 GMT
x-goog-storage-class: REGIONAL
accept-ranges: bytes
content-length: 17509
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ echo 'Drop collection'
Drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0EDUbvEm4L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D6WjyvmW8p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0EDUbvEm4L
++ cat /tmp/tmp.D6WjyvmW8p
++ rm /tmp/tmp.0EDUbvEm4L /tmp/tmp.D6WjyvmW8p
++ return 0
+ local client_container=psmdb-client-8f86f7874-5s278
+ kubectl_bin exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rpKdgEreVi
++ mktemp
+ local LAST_ERR=/tmp/tmp.0OXMFMbTdc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-8f86f7874-5s278 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rpKdgEreVi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6e33ebcd-9655-4e11-8421-6623c2b5a808") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.0OXMFMbTdc
+ rm /tmp/tmp.rpKdgEreVi /tmp/tmp.0OXMFMbTdc
+ return 0
+ echo 'Check backup and restore -- gcp-cs-sa'
Check backup and restore -- gcp-cs-sa
+ run_restore backup-gcp-cs-sa-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa-sharded/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.O31eRu1Eeq
++ mktemp
+ local LAST_ERR=/tmp/tmp.hMpqZDkbhx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.O31eRu1Eeq
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa-sharded created
+ cat /tmp/tmp.hMpqZDkbhx
+ rm /tmp/tmp.O31eRu1Eeq /tmp/tmp.hMpqZDkbhx
+ return 0
+ run_recovery_check backup-gcp-cs-sa-sharded _restore_sharded sharded
+ local backup_name=backup-gcp-cs-sa-sharded
+ local compare_suffix=_restore_sharded
+ local is_sharded=sharded
+ local tls=
+ wait_restore backup-gcp-cs-sa-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sa-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa-sharded to reach state "requested" .....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.UCrNrKEpwi/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-gcp-native-12994", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FwnrLzDpbq
++ mktemp
+ local LAST_ERR=/tmp/tmp.TQyZXpVnbH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FwnrLzDpbq
+ cat /tmp/tmp.TQyZXpVnbH
+ rm /tmp/tmp.FwnrLzDpbq /tmp/tmp.TQyZXpVnbH
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.UCrNrKEpwi/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.UCrNrKEpwi/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.UCrNrKEpwi/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1850/e2e-tests/demand-backup-physical-sharded-gcp-native/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.UCrNrKEpwi/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-11-10T07:42:05+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ wait_restore backup-gcp-cs-sa-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sa-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sa-sharded object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs-sa-sharded to reach state "ready" ..................................................

-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-0
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.CRRedEuBCN
++ mktemp
+ local LAST_ERR=/tmp/tmp.tlvtQ3mDcP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CRRedEuBCN
2025/11/10 07:41:47 [entrypoint] starting `pbm-agent`
2025-11-10T07:41:51.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-10T07:41:51.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-10T07:41:51.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-10T07:41:51.000+0000 I starting PITR routine
2025-11-10T07:41:51.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017
2025-11-10T07:41:51.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-10T07:41:51.000+0000 I listening for the commands
2025-11-10T07:42:05.000+0000 I got command restore [name: 2025-11-10T07:42:04.746443158Z, snapshot: 2025-11-10T07:37:02Z] <ts: 1762760524>, opid: 6911974c5c9831d655d948a8
2025-11-10T07:42:05.000+0000 I got epoch {1762760524 6}
2025-11-10T07:42:05.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] backup: 2025-11-10T07:37:02Z
2025-11-10T07:42:05.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] recovery started
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] port: 27245
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] wait for cluster status
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:06.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.tlvtQ3mDcP
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.CRRedEuBCN /tmp/tmp.tlvtQ3mDcP
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-1
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.7bwVYHQfrf
++ mktemp
+ local LAST_ERR=/tmp/tmp.zGNG3k1y7P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7bwVYHQfrf
2025/11/10 07:41:04 [entrypoint] starting `pbm-agent`
2025-11-10T07:41:07.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-10T07:41:07.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-10T07:41:07.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-10T07:41:07.000+0000 I starting PITR routine
2025-11-10T07:41:07.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017
2025-11-10T07:41:07.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-10T07:41:07.000+0000 I listening for the commands
2025-11-10T07:42:04.000+0000 I got command restore [name: 2025-11-10T07:42:04.746443158Z, snapshot: 2025-11-10T07:37:02Z] <ts: 1762760524>, opid: 6911974c5c9831d655d948a8
2025-11-10T07:42:04.000+0000 I got epoch {1762760524 6}
2025-11-10T07:42:05.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] backup: 2025-11-10T07:37:02Z
2025-11-10T07:42:05.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] recovery started
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] port: 27542
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] wait for cluster status
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.zGNG3k1y7P
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.7bwVYHQfrf /tmp/tmp.zGNG3k1y7P
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-2
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ua4HD5sgl
++ mktemp
+ local LAST_ERR=/tmp/tmp.pEO5rePWVQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8ua4HD5sgl
2025/11/10 07:40:29 [entrypoint] starting `pbm-agent`
2025-11-10T07:40:32.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2025-11-10T07:40:32.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2025-11-10T07:40:32.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2025-11-10T07:40:32.000+0000 I starting PITR routine
2025-11-10T07:40:32.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017
2025-11-10T07:40:32.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2025-11-10T07:40:32.000+0000 I listening for the commands
2025-11-10T07:42:04.000+0000 I got command restore [name: 2025-11-10T07:42:04.746443158Z, snapshot: 2025-11-10T07:37:02Z] <ts: 1762760524>, opid: 6911974c5c9831d655d948a8
2025-11-10T07:42:04.000+0000 I got epoch {1762760276 1}
2025-11-10T07:42:04.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] oplog slicer disabled
2025-11-10T07:42:04.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] backup: 2025-11-10T07:37:02Z
2025-11-10T07:42:04.000+0000 I [restore/2025-11-10T07:42:04.746443158Z] recovery started
2025-11-10T07:42:04.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] port: 27371
2025-11-10T07:42:04.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:04.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:42:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] wait for cluster status
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:44:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:46:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:48:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:50:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:52:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:54:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:56:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T07:58:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:00:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:02:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:04:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:06:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:08:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:10:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:12:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:14:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:16:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:18:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:20:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:22:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:24:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:26:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:28:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:30:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-gcp-native-12994.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
2025-11-10T08:32:05.000+0000 D [restore/2025-11-10T07:42:04.746443158Z] uploading ".pbm.restore/2025-11-10T07:42:04.746443158Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)]
+ cat /tmp/tmp.pEO5rePWVQ
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.8ua4HD5sgl /tmp/tmp.pEO5rePWVQ
+ return 0
+ grep restore-backup-gcp-cs-sa-sharded
+ tail -100
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U06OcqULY4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YSiKfa8xVh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U06OcqULY4
++ cat /tmp/tmp.YSiKfa8xVh
++ rm /tmp/tmp.U06OcqULY4 /tmp/tmp.YSiKfa8xVh
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-67bdb49f8b-kl79d
++ mktemp
+ local LAST_OUT=/tmp/tmp.2d76WUGVot
++ mktemp
+ local LAST_ERR=/tmp/tmp.j0LL1V1LUd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-67bdb49f8b-kl79d
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2d76WUGVot
+ cat /tmp/tmp.j0LL1V1LUd
+ rm /tmp/tmp.2d76WUGVot /tmp/tmp.j0LL1V1LUd
+ return 0
2025-11-10T08:31:49.109Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8ed8f4a6-be2a-4329-a440-e7ec6dceab8d", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:31:49.109Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8ed8f4a6-be2a-4329-a440-e7ec6dceab8d", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:31:49.109Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8ed8f4a6-be2a-4329-a440-e7ec6dceab8d", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:31:49.978Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8ed8f4a6-be2a-4329-a440-e7ec6dceab8d", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:31:54.979Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:31:54.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "pod": "some-name-rs0-0"}
2025-11-10T08:31:54.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "pod": "some-name-rs0-1"}
2025-11-10T08:31:54.981Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "pod": "some-name-rs0-2"}
2025-11-10T08:31:54.981Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:31:54.981Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:31:54.981Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:31:56.112Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "1c8810af-502e-497d-9226-1a9afe7ba169", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:01.113Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:01.114Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "pod": "some-name-rs0-0"}
2025-11-10T08:32:01.114Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "pod": "some-name-rs0-1"}
2025-11-10T08:32:01.114Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "pod": "some-name-rs0-2"}
2025-11-10T08:32:01.114Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:01.115Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:01.115Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:02.258Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "9d14f948-3215-45f8-b164-2448ae8d7ec5", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:07.259Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:07.261Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "pod": "some-name-rs0-0"}
2025-11-10T08:32:07.261Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "pod": "some-name-rs0-1"}
2025-11-10T08:32:07.261Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "pod": "some-name-rs0-2"}
2025-11-10T08:32:07.261Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:07.261Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:07.261Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:08.524Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "3b1ef508-6226-4c52-b01d-3beb505b372e", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:13.525Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:13.526Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "pod": "some-name-rs0-2"}
2025-11-10T08:32:13.526Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "pod": "some-name-rs0-0"}
2025-11-10T08:32:13.526Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "pod": "some-name-rs0-1"}
2025-11-10T08:32:13.526Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:13.527Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:13.527Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:14.710Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c435b09f-38f6-47e5-aa32-26c41e8a624d", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:19.711Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:19.712Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "pod": "some-name-rs0-0"}
2025-11-10T08:32:19.712Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "pod": "some-name-rs0-1"}
2025-11-10T08:32:19.712Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "pod": "some-name-rs0-2"}
2025-11-10T08:32:19.712Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:19.712Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:19.712Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:20.917Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "8023c967-36f7-4865-8aea-24260d2d07f2", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:25.918Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:25.919Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "pod": "some-name-rs0-0"}
2025-11-10T08:32:25.919Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "pod": "some-name-rs0-1"}
2025-11-10T08:32:25.919Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "pod": "some-name-rs0-2"}
2025-11-10T08:32:25.919Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:25.919Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:25.919Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:27.081Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c365a493-7595-46e4-bf32-1c53de0ef4c7", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:32.082Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:32.083Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "pod": "some-name-rs0-0"}
2025-11-10T08:32:32.083Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "pod": "some-name-rs0-1"}
2025-11-10T08:32:32.083Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "pod": "some-name-rs0-2"}
2025-11-10T08:32:32.083Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:32.083Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:32.083Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:33.223Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b3ebee8b-066a-4571-a625-133db3653be3", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:38.224Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:38.226Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "pod": "some-name-rs0-0"}
2025-11-10T08:32:38.226Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "pod": "some-name-rs0-1"}
2025-11-10T08:32:38.226Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "pod": "some-name-rs0-2"}
2025-11-10T08:32:38.226Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:38.226Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:38.226Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:39.110Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b7801c1f-46f7-4c43-aa55-9eb46221b237", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:44.111Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:44.112Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "pod": "some-name-rs0-0"}
2025-11-10T08:32:44.112Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "pod": "some-name-rs0-1"}
2025-11-10T08:32:44.112Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "pod": "some-name-rs0-2"}
2025-11-10T08:32:44.112Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:44.112Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:44.112Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:45.193Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b9920622-0532-4288-84cf-2fdd249c6dcc", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:50.194Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:50.197Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "pod": "some-name-rs0-0"}
2025-11-10T08:32:50.198Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "pod": "some-name-rs0-1"}
2025-11-10T08:32:50.198Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "pod": "some-name-rs0-2"}
2025-11-10T08:32:50.198Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:50.198Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:50.198Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:51.101Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "5fcda651-2121-4671-9a48-2e5b055c18d7", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:32:56.102Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:32:56.104Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "pod": "some-name-rs0-0"}
2025-11-10T08:32:56.104Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "pod": "some-name-rs0-1"}
2025-11-10T08:32:56.104Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "pod": "some-name-rs0-2"}
2025-11-10T08:32:56.104Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:32:56.105Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:32:56.105Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:32:57.234Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "b4402d42-a373-4451-bf60-8f4d75dea422", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
2025-11-10T08:33:02.235Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-gcp-native-12994"}
2025-11-10T08:33:02.236Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "pod": "some-name-rs0-0"}
2025-11-10T08:33:02.236Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "pod": "some-name-rs0-1"}
2025-11-10T08:33:02.236Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "pod": "some-name-rs0-2"}
2025-11-10T08:33:02.236Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "sts": "some-name-rs0", "replset": "rs0"}
2025-11-10T08:33:02.236Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "sts": "some-name-cfg", "replset": "cfg"}
2025-11-10T08:33:02.236Z	DEBUG	Check restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T07:42:04.746443158Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"}
2025-11-10T08:33:03.365Z	DEBUG	PBM restore status	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"}, "namespace": "demand-backup-physical-sharded-gcp-native-12994", "name": "restore-backup-gcp-cs-sa-sharded", "reconcileID": "c346e980-230a-41a9-958c-21deaa913aee", "status": {"type":"physical","opid":"","name":"2025-11-10T07:42:04.746443158Z","replsets":[{"name":"cfg","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}}
+ kubectl_bin get psmdb-restore restore-backup-gcp-cs-sa-sharded -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.04JqwvUuYs
++ mktemp
+ local LAST_ERR=/tmp/tmp.KGhayVRWNu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-restore restore-backup-gcp-cs-sa-sharded -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.04JqwvUuYs
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-gcp-cs-sa-sharded","namespace":"demand-backup-physical-sharded-gcp-native-12994"},"spec":{"backupName":"backup-gcp-cs-sa-sharded","clusterName":"some-name"}}
  creationTimestamp: "2025-11-10T07:37:35Z"
  generation: 1
  name: restore-backup-gcp-cs-sa-sharded
  namespace: demand-backup-physical-sharded-gcp-native-12994
  resourceVersion: "1762760524787935000"
  uid: 471bf91e-1ac6-4610-9a95-6fab39e962a1
spec:
  backupName: backup-gcp-cs-sa-sharded
  clusterName: some-name
status:
  pbmName: "2025-11-10T07:42:04.746443158Z"
  state: requested
+ cat /tmp/tmp.KGhayVRWNu
+ rm /tmp/tmp.04JqwvUuYs /tmp/tmp.KGhayVRWNu
+ return 0
+ kubectl_bin describe psmdb-restore restore-backup-gcp-cs-sa-sharded
++ mktemp
+ local LAST_OUT=/tmp/tmp.iY5I34ipFG
++ mktemp
+ local LAST_ERR=/tmp/tmp.VwvLdQhwBj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl describe psmdb-restore restore-backup-gcp-cs-sa-sharded
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iY5I34ipFG
Name:         restore-backup-gcp-cs-sa-sharded
Namespace:    demand-backup-physical-sharded-gcp-native-12994
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2025-11-10T07:37:35Z
  Generation:          1
  Resource Version:    1762760524787935000
  UID:                 471bf91e-1ac6-4610-9a95-6fab39e962a1
Spec:
  Backup Name:   backup-gcp-cs-sa-sharded
  Cluster Name:  some-name
Status:
  Pbm Name:  2025-11-10T07:42:04.746443158Z
  State:     requested
Events:      <none>
+ cat /tmp/tmp.VwvLdQhwBj
+ rm /tmp/tmp.iY5I34ipFG /tmp/tmp.VwvLdQhwBj
+ return 0
+ echo 'Restore object restore-backup-gcp-cs-sa-sharded is in requested state after 50 minutes.'
Restore object restore-backup-gcp-cs-sa-sharded is in requested state after 50 minutes.
+ echo something went wrong with operator or kubernetes cluster
something went wrong with operator or kubernetes cluster
+ exit 1