Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/logs/demand-backup-physical-gcp-native.log grep: warning: stray \ before - Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + '[' -n '' ']' + create_infra demand-backup-physical-gcp-native-18874 + local ns=demand-backup-physical-gcp-native-18874 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.uCk7OJk1TE ++ mktemp + local LAST_ERR=/tmp/tmp.uocoQpV6lY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uCk7OJk1TE customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.uocoQpV6lY + rm /tmp/tmp.uCk7OJk1TE /tmp/tmp.uocoQpV6lY + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.cHvggNOgsI ++ mktemp + local LAST_ERR=/tmp/tmp.9CZ9fK2Yxx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cHvggNOgsI + cat /tmp/tmp.9CZ9fK2Yxx + rm /tmp/tmp.cHvggNOgsI /tmp/tmp.9CZ9fK2Yxx + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.jl665YM5gm ++ mktemp + local LAST_ERR=/tmp/tmp.tbFNrMIvNu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jl665YM5gm + cat /tmp/tmp.tbFNrMIvNu + rm /tmp/tmp.jl665YM5gm /tmp/tmp.tbFNrMIvNu + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.hODqMV5pw6 ++ mktemp + local LAST_ERR=/tmp/tmp.47SKyfPgrq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hODqMV5pw6 + cat /tmp/tmp.47SKyfPgrq + rm /tmp/tmp.hODqMV5pw6 /tmp/tmp.47SKyfPgrq + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.pSYyghFABp ++ mktemp + local LAST_ERR=/tmp/tmp.J9eQI01TG0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pSYyghFABp clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.J9eQI01TG0 + rm /tmp/tmp.pSYyghFABp /tmp/tmp.J9eQI01TG0 + return 0 + check_crd_for_deletion PR-2105-619f8699 + local git_tag=PR-2105-619f8699 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2105-619f8699/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Sck3ki1unQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.tReLe2Soby ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Sck3ki1unQ ++ cat /tmp/tmp.tReLe2Soby Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Sck3ki1unQ ++ cat /tmp/tmp.tReLe2Soby Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Sck3ki1unQ ++ cat /tmp/tmp.tReLe2Soby Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Sck3ki1unQ ++ cat /tmp/tmp.tReLe2Soby Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Sck3ki1unQ /tmp/tmp.tReLe2Soby ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.uzTQqeHGvu + local LAST_OUT=/tmp/tmp.ffRYUiwpy3 ++ mktemp egrep: warning: egrep is obsolescent; using grep -E ++ mktemp + local LAST_ERR=/tmp/tmp.RrJwNJHwAC + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.boUHPDhNRI + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uzTQqeHGvu + cat /tmp/tmp.RrJwNJHwAC + rm /tmp/tmp.uzTQqeHGvu /tmp/tmp.RrJwNJHwAC + return 0 namespace "demand-backup-physical-gcp-native-21684" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ffRYUiwpy3 namespace "psmdb-operator" deleted + cat /tmp/tmp.boUHPDhNRI + rm /tmp/tmp.ffRYUiwpy3 /tmp/tmp.boUHPDhNRI + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.KcaPvHtG9N ++ mktemp + local LAST_ERR=/tmp/tmp.nnGpnAkNzb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KcaPvHtG9N + cat /tmp/tmp.nnGpnAkNzb + rm /tmp/tmp.KcaPvHtG9N /tmp/tmp.nnGpnAkNzb + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.y2vem6PkZU ++ mktemp + local LAST_ERR=/tmp/tmp.Xo8rea5SGn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.y2vem6PkZU namespace/psmdb-operator created + cat /tmp/tmp.Xo8rea5SGn + rm /tmp/tmp.y2vem6PkZU /tmp/tmp.Xo8rea5SGn + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.TxGaaBj1pw +++ mktemp ++ local LAST_ERR=/tmp/tmp.95p15UVzeQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TxGaaBj1pw ++ cat /tmp/tmp.95p15UVzeQ ++ rm /tmp/tmp.TxGaaBj1pw /tmp/tmp.95p15UVzeQ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.k6aODNRfLF ++ mktemp + local LAST_ERR=/tmp/tmp.llzQH4T3uw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.k6aODNRfLF Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3" modified. + cat /tmp/tmp.llzQH4T3uw + rm /tmp/tmp.k6aODNRfLF /tmp/tmp.llzQH4T3uw + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2105-619f8699' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2105-619f8699 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.u6exrbXEVO ++ mktemp + local LAST_ERR=/tmp/tmp.DebtbvvOst + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u6exrbXEVO customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.DebtbvvOst + rm /tmp/tmp.u6exrbXEVO /tmp/tmp.DebtbvvOst + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.a9p1FBWjEu ++ mktemp + local LAST_ERR=/tmp/tmp.3iKZtpAavM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a9p1FBWjEu clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.3iKZtpAavM + rm /tmp/tmp.a9p1FBWjEu /tmp/tmp.3iKZtpAavM + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2105-619f8699") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.cl3bA8GcA9 ++ mktemp + local LAST_ERR=/tmp/tmp.et3MXofxt5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cl3bA8GcA9 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.et3MXofxt5 + rm /tmp/tmp.cl3bA8GcA9 /tmp/tmp.et3MXofxt5 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.OXHNt1LzjF +++ mktemp ++ local LAST_ERR=/tmp/tmp.fcyAYh9ua2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OXHNt1LzjF ++ cat /tmp/tmp.fcyAYh9ua2 ++ rm /tmp/tmp.OXHNt1LzjF /tmp/tmp.fcyAYh9ua2 ++ return 0 + wait_operator_pod percona-server-mongodb-operator-cd5c496f-ckn4r + local pod=percona-server-mongodb-operator-cd5c496f-ckn4r + set +o xtrace waiting for pod/percona-server-mongodb-operator-cd5c496f-ckn4r to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mm1yxQvpDd +++ mktemp ++ local LAST_ERR=/tmp/tmp.nRx14IYru6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Mm1yxQvpDd ++ cat /tmp/tmp.nRx14IYru6 ++ rm /tmp/tmp.Mm1yxQvpDd /tmp/tmp.nRx14IYru6 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-cd5c496f-ckn4r ++ mktemp + local LAST_OUT=/tmp/tmp.0LS5ysBjBM ++ mktemp + local LAST_ERR=/tmp/tmp.kpUrRHFQxO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-cd5c496f-ckn4r + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0LS5ysBjBM + cat /tmp/tmp.kpUrRHFQxO + rm /tmp/tmp.0LS5ysBjBM /tmp/tmp.kpUrRHFQxO + return 0 2025-11-10T02:49:29.175Z INFO setup Manager starting up {"gitCommit": "619f8699cd8c6a6ba4c84f5922b01cb9941bafbd", "gitBranch": "PR-2105-619f8699", "buildTime": "", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-gcp-native-18874 + local namespace=demand-backup-physical-gcp-native-18874 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces demand-backup-physical-gcp-native-18874' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-gcp-native-18874 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-gcp-native-18874 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.KzWENsyEo0 egrep: warning: egrep is obsolescent; using grep -E + local LAST_OUT=/tmp/tmp.Wwyrw2LDjo ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.mSDHx1ag3u + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.sWdBEZRdtE + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-physical-gcp-native-18874 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KzWENsyEo0 + cat /tmp/tmp.mSDHx1ag3u + rm /tmp/tmp.KzWENsyEo0 /tmp/tmp.mSDHx1ag3u + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Wwyrw2LDjo + cat /tmp/tmp.sWdBEZRdtE + rm /tmp/tmp.Wwyrw2LDjo /tmp/tmp.sWdBEZRdtE + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-gcp-native-18874 ++ mktemp + local LAST_OUT=/tmp/tmp.zqdAZqC3bP ++ mktemp + local LAST_ERR=/tmp/tmp.w8RODkJHnF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-physical-gcp-native-18874 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zqdAZqC3bP + cat /tmp/tmp.w8RODkJHnF + rm /tmp/tmp.zqdAZqC3bP /tmp/tmp.w8RODkJHnF + return 0 + desc 'create namespace demand-backup-physical-gcp-native-18874' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-gcp-native-18874 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-gcp-native-18874 ++ mktemp + local LAST_OUT=/tmp/tmp.fhpM69JdsD ++ mktemp + local LAST_ERR=/tmp/tmp.3GiHHLeXO9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-physical-gcp-native-18874 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fhpM69JdsD namespace/demand-backup-physical-gcp-native-18874 created + cat /tmp/tmp.3GiHHLeXO9 + rm /tmp/tmp.fhpM69JdsD /tmp/tmp.3GiHHLeXO9 + return 0 + set_kube_ctx demand-backup-physical-gcp-native-18874 + local namespace=demand-backup-physical-gcp-native-18874 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.WC37eYqsMP +++ mktemp ++ local LAST_ERR=/tmp/tmp.RJUiB8QTNx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WC37eYqsMP ++ cat /tmp/tmp.RJUiB8QTNx ++ rm /tmp/tmp.WC37eYqsMP /tmp/tmp.RJUiB8QTNx ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3 --namespace=demand-backup-physical-gcp-native-18874 ++ mktemp + local LAST_OUT=/tmp/tmp.obFxlqUIiD ++ mktemp + local LAST_ERR=/tmp/tmp.BtJgfZr8IZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3 --namespace=demand-backup-physical-gcp-native-18874 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.obFxlqUIiD Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2105-619f8699-1-cluster3" modified. + cat /tmp/tmp.BtJgfZr8IZ + rm /tmp/tmp.obFxlqUIiD /tmp/tmp.BtJgfZr8IZ + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Idu5c9wrps ++ mktemp + local LAST_ERR=/tmp/tmp.QeRvHDxWVP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Idu5c9wrps secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.QeRvHDxWVP + rm /tmp/tmp.Idu5c9wrps /tmp/tmp.QeRvHDxWVP + return 0 + desc 'Testing on not sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on not sharded cluster ----------------------------------------------------------------------------------- + echo 'Creating PSMDB cluster' Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.WUK8MI9cyt ++ mktemp + local LAST_ERR=/tmp/tmp.wEOJjisLXU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WUK8MI9cyt secret/some-users created + cat /tmp/tmp.wEOJjisLXU + rm /tmp/tmp.WUK8MI9cyt /tmp/tmp.wEOJjisLXU + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2105-619f8699"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.tttJ3k7KXd + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.KdyV925uo8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tttJ3k7KXd perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.KdyV925uo8 + rm /tmp/tmp.tttJ3k7KXd /tmp/tmp.KdyV925uo8 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.F97dUu6kPa ++ mktemp + local LAST_ERR=/tmp/tmp.7j5DZOZtzR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F97dUu6kPa deployment.apps/psmdb-client created + cat /tmp/tmp.7j5DZOZtzR + rm /tmp/tmp.F97dUu6kPa /tmp/tmp.7j5DZOZtzR + return 0 + echo 'Check if all pods started' Check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0Q2rJrYrNk +++ mktemp ++ local LAST_ERR=/tmp/tmp.vpu00rCnmW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0Q2rJrYrNk ++ cat /tmp/tmp.vpu00rCnmW ++ rm /tmp/tmp.0Q2rJrYrNk /tmp/tmp.vpu00rCnmW ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bZboN7dktL +++ mktemp ++ local LAST_ERR=/tmp/tmp.tSnhg6h1JT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bZboN7dktL ++ cat /tmp/tmp.tSnhg6h1JT ++ rm /tmp/tmp.bZboN7dktL /tmp/tmp.tSnhg6h1JT ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xbNMbCnHJC +++ mktemp ++ local LAST_ERR=/tmp/tmp.pWV4CLaH51 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xbNMbCnHJC ++ cat /tmp/tmp.pWV4CLaH51 ++ rm /tmp/tmp.xbNMbCnHJC /tmp/tmp.pWV4CLaH51 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tlzEQthG0w +++ mktemp ++ local LAST_ERR=/tmp/tmp.Xznod1cg86 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tlzEQthG0w ++ cat /tmp/tmp.Xznod1cg86 ++ rm /tmp/tmp.tlzEQthG0w /tmp/tmp.Xznod1cg86 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + echo 'Writing test data' Writing test data + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-18874 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hLzh1Ks7XF +++ mktemp ++ local LAST_ERR=/tmp/tmp.0v8ofSpH8Q ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hLzh1Ks7XF ++ cat /tmp/tmp.0v8ofSpH8Q ++ rm /tmp/tmp.hLzh1Ks7XF /tmp/tmp.0v8ofSpH8Q ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.arGGzflms2 ++ mktemp + local LAST_ERR=/tmp/tmp.QCz8HUETEY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.arGGzflms2 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1d61397a-42a4-4ee5-8ed3-26c58f210a80") } Percona Server for MongoDB server version: v8.0.12-4 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.QCz8HUETEY + rm /tmp/tmp.arGGzflms2 /tmp/tmp.QCz8HUETEY + return 0 + sleep 1 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uhrgFa8Fso +++ mktemp ++ local LAST_ERR=/tmp/tmp.LK8PXycoQf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uhrgFa8Fso ++ cat /tmp/tmp.LK8PXycoQf ++ rm /tmp/tmp.uhrgFa8Fso /tmp/tmp.LK8PXycoQf ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.3AjU1LGhrB ++ mktemp + local LAST_ERR=/tmp/tmp.WenlTS1liQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3AjU1LGhrB Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("a1d2bf87-a1a0-4332-b611-48d726a4972c") } Percona Server for MongoDB server version: v8.0.12-4 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.WenlTS1liQ + rm /tmp/tmp.3AjU1LGhrB /tmp/tmp.WenlTS1liQ + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-11-10T02:52:50+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z7T9A742wV egrep: warning: egrep is obsolescent; using grep -E +++ mktemp ++ local LAST_ERR=/tmp/tmp.0uUNVfWDQP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z7T9A742wV ++ cat /tmp/tmp.0uUNVfWDQP ++ rm /tmp/tmp.z7T9A742wV /tmp/tmp.0uUNVfWDQP ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.E7IKSSAHeW ++ mktemp + local LAST_ERR=/tmp/tmp.gleTP5L0qX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E7IKSSAHeW + cat /tmp/tmp.gleTP5L0qX + rm /tmp/tmp.E7IKSSAHeW /tmp/tmp.gleTP5L0qX + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.A70ptfS0n1/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-11-10T02:52:53+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' egrep: warning: egrep is obsolescent; using grep -E +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ir3tFIBpvF +++ mktemp ++ local LAST_ERR=/tmp/tmp.nu3iitN6QQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ir3tFIBpvF ++ cat /tmp/tmp.nu3iitN6QQ ++ rm /tmp/tmp.Ir3tFIBpvF /tmp/tmp.nu3iitN6QQ ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.eDFLKLsU55 ++ mktemp + local LAST_ERR=/tmp/tmp.kFj64eDLja + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eDFLKLsU55 + cat /tmp/tmp.kFj64eDLja + rm /tmp/tmp.eDFLKLsU55 /tmp/tmp.kFj64eDLja + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.A70ptfS0n1/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-11-10T02:52:55+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp egrep: warning: egrep is obsolescent; using grep -E ++ local LAST_OUT=/tmp/tmp.OoQU45cM11 +++ mktemp ++ local LAST_ERR=/tmp/tmp.M0EN0OKpUR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OoQU45cM11 ++ cat /tmp/tmp.M0EN0OKpUR ++ rm /tmp/tmp.OoQU45cM11 /tmp/tmp.M0EN0OKpUR ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.S9cthG8s9m ++ mktemp + local LAST_ERR=/tmp/tmp.7tRLj99Dz4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S9cthG8s9m + cat /tmp/tmp.7tRLj99Dz4 + rm /tmp/tmp.S9cthG8s9m /tmp/tmp.7tRLj99Dz4 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.A70ptfS0n1/find + echo 'Running GCP backups' Running GCP backups + backup_name_gcp=backup-gcp-cs-sa + run_backup gcp-cs-sa backup-gcp-cs-sa physical + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=physical + desc 'run backup backup-gcp-cs-sa' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-gcp-cs-sa ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-gcp-cs-sa" | .spec.storageName = "gcp-cs-sa" | .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WzXxBDvfG5 ++ mktemp + local LAST_ERR=/tmp/tmp.eKjOmPEyBN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WzXxBDvfG5 perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.eKjOmPEyBN + rm /tmp/tmp.WzXxBDvfG5 /tmp/tmp.eKjOmPEyBN + return 0 + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state...........OK + check_backup_in_storage backup-gcp-cs-sa gcs rs0 + local backup=backup-gcp-cs-sa + local storage_type=gcs + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=storage.googleapis.com ++ get_backup_dest backup-gcp-cs-sa ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-gcp-cs-sa ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ local LAST_OUT=/tmp/tmp.oZnK8srtjo ++ sed 's|gs://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.wfaSrudhkl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oZnK8srtjo ++ cat /tmp/tmp.wfaSrudhkl ++ rm /tmp/tmp.oZnK8srtjo /tmp/tmp.wfaSrudhkl ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical/2025-11-10T02:53:00Z + [[ gcs == \m\i\n\i\o ]] + local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical/2025-11-10T02:53:00Z/rs0/filelist.pbm + log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical/2025-11-10T02:53:00Z/rs0/filelist.pbm exists' + set +o xtrace [2025-11-10T02:53:22+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical/2025-11-10T02:53:00Z/rs0/filelist.pbm exists + curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical/2025-11-10T02:53:00Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 12363 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/2 200 content-type: application/octet-stream x-guploader-uploadid: AOCedOE_mvaXDkhjLUzYs7dvt7ErsMZy8X_7zmrPL6YgpYVEhHU_9ghkMiPuvvNfJ029oI8 expires: Mon, 10 Nov 2025 03:53:22 GMT date: Mon, 10 Nov 2025 02:53:22 GMT cache-control: public, max-age=3600 last-modified: Mon, 10 Nov 2025 02:53:14 GMT etag: "d9d59d2e8d4d7b4795f75277533619e9" x-goog-generation: 1762743194718951 x-goog-metageneration: 1 x-goog-stored-content-encoding: identity x-goog-stored-content-length: 12363 x-goog-hash: crc32c=LELeeQ== x-goog-hash: md5=2dWdLo1Ne0eV91J3UzYZ6Q== x-goog-expiration: Tue, 11 Nov 2025 02:53:14 GMT x-goog-storage-class: REGIONAL accept-ranges: bytes content-length: 12363 server: UploadServer alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + echo 'Drop collection' Drop collection + run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yFe3lwFIfP +++ mktemp ++ local LAST_ERR=/tmp/tmp.HD1kvybug4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yFe3lwFIfP ++ cat /tmp/tmp.HD1kvybug4 ++ rm /tmp/tmp.yFe3lwFIfP /tmp/tmp.HD1kvybug4 ++ return 0 + local client_container=psmdb-client-8f86f7874-6gspm + local mongo_flag= + [[ myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.GwlrgrryLO ++ mktemp + local LAST_ERR=/tmp/tmp.RQl1jKa92h + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-8f86f7874-6gspm -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GwlrgrryLO Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("fb37a606-89f4-4fe0-be0b-28d3368a49d7") } Percona Server for MongoDB server version: v8.0.12-4 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.RQl1jKa92h + rm /tmp/tmp.GwlrgrryLO /tmp/tmp.RQl1jKa92h + return 0 + echo 'check backup and restore -- gcp-cs-sa' check backup and restore -- gcp-cs-sa + run_restore backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/conf/restore.yml + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/' + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.25N4IGhO1q ++ mktemp + local LAST_ERR=/tmp/tmp.2x3nt7wl0l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.25N4IGhO1q perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created + cat /tmp/tmp.2x3nt7wl0l + rm /tmp/tmp.25N4IGhO1q /tmp/tmp.2x3nt7wl0l + return 0 + run_recovery_check backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local compare_suffix=_restore + local is_sharded= + local tls= + wait_restore backup-gcp-cs-sa some-name requested 0 3000 + local backup_name=backup-gcp-cs-sa + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "requested" ...OK after 2 minutes + [[ 0 -eq 1 ]] + echo + compare_kubectl statefulset/some-name-rs0 _restore + local resource=statefulset/some-name-rs0 + local postfix=_restore + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/statefulset_some-name-rs0_restore.yml + local new_result=/tmp/tmp.A70ptfS0n1/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/statefulset_some-name-rs0_restore-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-physical-gcp-native-18874", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.bAAIDlFvMI ++ mktemp + local LAST_ERR=/tmp/tmp.4DgjIxoCLf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bAAIDlFvMI + cat /tmp/tmp.4DgjIxoCLf + rm /tmp/tmp.bAAIDlFvMI /tmp/tmp.4DgjIxoCLf + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.A70ptfS0n1/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.31 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.A70ptfS0n1/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.A70ptfS0n1/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/statefulset_some-name-rs0_restore.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2105/e2e-tests/demand-backup-physical-gcp-native/compare/statefulset_some-name-rs0_restore.yml /tmp/tmp.A70ptfS0n1/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2025-11-10T02:55:38+0000] compare_kubectl: statefulset/some-name-rs0 OK + wait_restore backup-gcp-cs-sa some-name ready 0 3000 + local backup_name=backup-gcp-cs-sa + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .................................................. ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.YXcsfqxtPe ++ mktemp + local LAST_ERR=/tmp/tmp.5p75Q4hf62 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YXcsfqxtPe 2025/11/10 02:55:22 [entrypoint] starting `pbm-agent` 2025-11-10T02:55:25.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-11-10T02:55:25.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-11-10T02:55:25.000+0000 I pbm-agent: Version: 2.12.0 Platform: linux/amd64 GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e GitBranch: release-2.12.0 BuildTime: 2025-10-31_09:21_UTC GoVersion: go1.25.3 2025-11-10T02:55:25.000+0000 I starting PITR routine 2025-11-10T02:55:25.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017 2025-11-10T02:55:25.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-11-10T02:55:25.000+0000 I listening for the commands 2025-11-10T02:55:37.000+0000 I got command restore [name: 2025-11-10T02:55:36.747221957Z, snapshot: 2025-11-10T02:53:00Z] , opid: 69115428f096c60dbe20e1a9 2025-11-10T02:55:37.000+0000 I got epoch {1762743209 2} 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] backup: 2025-11-10T02:53:00Z 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] recovery started 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] port: 27953 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] wait for cluster status 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] + cat /tmp/tmp.5p75Q4hf62 Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.YXcsfqxtPe /tmp/tmp.5p75Q4hf62 + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.Vgb6QlIuJe ++ mktemp + local LAST_ERR=/tmp/tmp.H4eIik1tsE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Vgb6QlIuJe 2025/11/10 02:54:42 [entrypoint] starting `pbm-agent` 2025-11-10T02:54:46.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-11-10T02:54:46.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-11-10T02:54:46.000+0000 I pbm-agent: Version: 2.12.0 Platform: linux/amd64 GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e GitBranch: release-2.12.0 BuildTime: 2025-10-31_09:21_UTC GoVersion: go1.25.3 2025-11-10T02:54:46.000+0000 I starting PITR routine 2025-11-10T02:54:46.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017 2025-11-10T02:54:46.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-11-10T02:54:46.000+0000 I listening for the commands 2025-11-10T02:55:37.000+0000 I got command restore [name: 2025-11-10T02:55:36.747221957Z, snapshot: 2025-11-10T02:53:00Z] , opid: 69115428f096c60dbe20e1a9 2025-11-10T02:55:37.000+0000 I got epoch {1762743209 2} 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] oplog slicer disabled 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] backup: 2025-11-10T02:53:00Z 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] recovery started 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] port: 27392 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] wait for cluster status 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] + cat /tmp/tmp.H4eIik1tsE Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.Vgb6QlIuJe /tmp/tmp.H4eIik1tsE + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.kiUZFp5l4n ++ mktemp + local LAST_ERR=/tmp/tmp.8JiJliwKkY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kiUZFp5l4n 2025/11/10 02:54:01 [entrypoint] starting `pbm-agent` 2025-11-10T02:54:04.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-11-10T02:54:04.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-11-10T02:54:04.000+0000 I pbm-agent: Version: 2.12.0 Platform: linux/amd64 GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e GitBranch: release-2.12.0 BuildTime: 2025-10-31_09:21_UTC GoVersion: go1.25.3 2025-11-10T02:54:04.000+0000 I starting PITR routine 2025-11-10T02:54:04.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017 2025-11-10T02:54:04.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-11-10T02:54:04.000+0000 I listening for the commands 2025-11-10T02:55:37.000+0000 I got command restore [name: 2025-11-10T02:55:36.747221957Z, snapshot: 2025-11-10T02:53:00Z] , opid: 69115428f096c60dbe20e1a9 2025-11-10T02:55:37.000+0000 I got epoch {1762743337 11} 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] backup: 2025-11-10T02:53:00Z 2025-11-10T02:55:37.000+0000 I [restore/2025-11-10T02:55:36.747221957Z] recovery started 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] port: 27576 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:55:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] wait for cluster status 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:57:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T02:59:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:01:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:03:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:05:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:07:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:09:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:11:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:13:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:15:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:17:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:19:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:21:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:23:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:25:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:27:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:29:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:31:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:33:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:35:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:37:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:39:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:41:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:43:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-18874.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2025-11-10T03:45:37.000+0000 D [restore/2025-11-10T02:55:36.747221957Z] uploading ".pbm.restore/2025-11-10T02:55:36.747221957Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] + cat /tmp/tmp.8JiJliwKkY Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.kiUZFp5l4n /tmp/tmp.8JiJliwKkY + return 0 + grep restore-backup-gcp-cs-sa + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.VUHi3St7OC +++ mktemp ++ local LAST_ERR=/tmp/tmp.oF7FXatWyq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VUHi3St7OC ++ cat /tmp/tmp.oF7FXatWyq ++ rm /tmp/tmp.VUHi3St7OC /tmp/tmp.oF7FXatWyq ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-cd5c496f-ckn4r ++ mktemp + local LAST_OUT=/tmp/tmp.UKSv2cn6Wm ++ mktemp + local LAST_ERR=/tmp/tmp.Yv14lPIthZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-cd5c496f-ckn4r + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UKSv2cn6Wm + cat /tmp/tmp.Yv14lPIthZ + rm /tmp/tmp.UKSv2cn6Wm /tmp/tmp.Yv14lPIthZ + return 0 2025-11-10T03:45:30.218Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "83a27866-9fdf-4377-aaa8-4e4142e44d99", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:30.799Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "83a27866-9fdf-4377-aaa8-4e4142e44d99", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:45:35.800Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:45:35.801Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "pod": "some-name-rs0-2"} 2025-11-10T03:45:35.801Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "pod": "some-name-rs0-0"} 2025-11-10T03:45:35.801Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "pod": "some-name-rs0-1"} 2025-11-10T03:45:35.801Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:45:35.801Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:36.475Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ee97fb0a-117b-4e92-9a6e-a2e87030a734", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:45:41.476Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:45:41.477Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "pod": "some-name-rs0-1"} 2025-11-10T03:45:41.477Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "pod": "some-name-rs0-2"} 2025-11-10T03:45:41.477Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "pod": "some-name-rs0-0"} 2025-11-10T03:45:41.477Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:45:41.477Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:42.262Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ec1f015b-81ef-429d-b4ea-a83d8a34b5cb", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:45:47.262Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:45:47.263Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "pod": "some-name-rs0-0"} 2025-11-10T03:45:47.263Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "pod": "some-name-rs0-1"} 2025-11-10T03:45:47.263Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "pod": "some-name-rs0-2"} 2025-11-10T03:45:47.263Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:45:47.263Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:47.830Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ed5f65be-fef7-43be-86f8-0e697e424095", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:45:52.830Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:45:52.831Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "pod": "some-name-rs0-0"} 2025-11-10T03:45:52.831Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "pod": "some-name-rs0-1"} 2025-11-10T03:45:52.831Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "pod": "some-name-rs0-2"} 2025-11-10T03:45:52.831Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:45:52.831Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:53.309Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3543c74-1309-4368-87d2-6bf707ec5694", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:45:58.310Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:45:58.311Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "pod": "some-name-rs0-1"} 2025-11-10T03:45:58.311Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "pod": "some-name-rs0-2"} 2025-11-10T03:45:58.311Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "pod": "some-name-rs0-0"} 2025-11-10T03:45:58.311Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:45:58.311Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:45:59.011Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d87b4d51-1897-4165-bde5-d12bd7c269a3", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:04.012Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:04.013Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "pod": "some-name-rs0-0"} 2025-11-10T03:46:04.013Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "pod": "some-name-rs0-1"} 2025-11-10T03:46:04.013Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "pod": "some-name-rs0-2"} 2025-11-10T03:46:04.013Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:04.013Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:04.505Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "de259366-16d7-4a30-8d0b-b65895e077e9", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:09.506Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:09.507Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "pod": "some-name-rs0-0"} 2025-11-10T03:46:09.507Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "pod": "some-name-rs0-1"} 2025-11-10T03:46:09.507Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "pod": "some-name-rs0-2"} 2025-11-10T03:46:09.507Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:09.507Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:10.201Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f664c81b-0977-43c2-945f-db1a32509bba", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:15.202Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:15.203Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "pod": "some-name-rs0-2"} 2025-11-10T03:46:15.203Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "pod": "some-name-rs0-0"} 2025-11-10T03:46:15.203Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "pod": "some-name-rs0-1"} 2025-11-10T03:46:15.203Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:15.203Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:15.905Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "65b649f8-a524-4875-9720-98db86d751aa", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:20.906Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:20.906Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "pod": "some-name-rs0-0"} 2025-11-10T03:46:20.906Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "pod": "some-name-rs0-1"} 2025-11-10T03:46:20.906Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "pod": "some-name-rs0-2"} 2025-11-10T03:46:20.906Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:20.906Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:21.598Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84326d66-b277-4781-9cb4-57930375e98a", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:26.600Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:26.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "pod": "some-name-rs0-0"} 2025-11-10T03:46:26.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "pod": "some-name-rs0-1"} 2025-11-10T03:46:26.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "pod": "some-name-rs0-2"} 2025-11-10T03:46:26.600Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:26.600Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:27.288Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "df6ee9b3-4360-4c90-9453-4a27dd6a30bd", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:32.288Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:32.289Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "pod": "some-name-rs0-0"} 2025-11-10T03:46:32.289Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "pod": "some-name-rs0-1"} 2025-11-10T03:46:32.289Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "pod": "some-name-rs0-2"} 2025-11-10T03:46:32.289Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:32.289Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:33.010Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "c11332bc-6391-43be-9026-7b292283bc62", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:38.011Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:38.012Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "pod": "some-name-rs0-0"} 2025-11-10T03:46:38.012Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "pod": "some-name-rs0-1"} 2025-11-10T03:46:38.012Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "pod": "some-name-rs0-2"} 2025-11-10T03:46:38.012Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:38.012Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:38.732Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6978d034-b2ec-4eed-a7cc-4892235fee5d", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:43.733Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:43.733Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "pod": "some-name-rs0-0"} 2025-11-10T03:46:43.733Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "pod": "some-name-rs0-1"} 2025-11-10T03:46:43.733Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "pod": "some-name-rs0-2"} 2025-11-10T03:46:43.733Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:43.733Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:44.489Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "113fdb57-fdb9-4235-a361-c6c31dd8a044", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-11-10T03:46:49.489Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-18874"} 2025-11-10T03:46:49.490Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "pod": "some-name-rs0-0"} 2025-11-10T03:46:49.490Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "pod": "some-name-rs0-1"} 2025-11-10T03:46:49.490Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "pod": "some-name-rs0-2"} 2025-11-10T03:46:49.490Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "sts": "some-name-rs0", "replset": "rs0"} 2025-11-10T03:46:49.490Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "command": ["/opt/percona/pbm", "describe-restore", "2025-11-10T02:55:36.747221957Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-11-10T03:46:50.186Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"}, "namespace": "demand-backup-physical-gcp-native-18874", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04c2d62a-9e90-419a-b0be-86f6bd1e21ed", "status": {"type":"physical","opid":"","name":"2025-11-10T02:55:36.747221957Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} + kubectl_bin get psmdb-restore restore-backup-gcp-cs-sa -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.3BZ1fY1z7X ++ mktemp + local LAST_ERR=/tmp/tmp.AhyTZKIqHu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-gcp-cs-sa -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3BZ1fY1z7X apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-18874"},"spec":{"backupName":"backup-gcp-cs-sa","clusterName":"some-name"}} creationTimestamp: "2025-11-10T02:53:28Z" generation: 1 name: restore-backup-gcp-cs-sa namespace: demand-backup-physical-gcp-native-18874 resourceVersion: "1762743336853615024" uid: abfe3151-ca8e-4c0a-b879-ee580b68112e spec: backupName: backup-gcp-cs-sa clusterName: some-name status: pbmName: "2025-11-10T02:55:36.747221957Z" state: requested + cat /tmp/tmp.AhyTZKIqHu + rm /tmp/tmp.3BZ1fY1z7X /tmp/tmp.AhyTZKIqHu + return 0 + kubectl_bin describe psmdb-restore restore-backup-gcp-cs-sa ++ mktemp + local LAST_OUT=/tmp/tmp.R5lmJiVoL2 ++ mktemp + local LAST_ERR=/tmp/tmp.bfH5m2WSFH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-gcp-cs-sa + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R5lmJiVoL2 Name: restore-backup-gcp-cs-sa Namespace: demand-backup-physical-gcp-native-18874 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2025-11-10T02:53:28Z Generation: 1 Resource Version: 1762743336853615024 UID: abfe3151-ca8e-4c0a-b879-ee580b68112e Spec: Backup Name: backup-gcp-cs-sa Cluster Name: some-name Status: Pbm Name: 2025-11-10T02:55:36.747221957Z State: requested Events: + cat /tmp/tmp.bfH5m2WSFH + rm /tmp/tmp.R5lmJiVoL2 /tmp/tmp.bfH5m2WSFH + return 0 + echo 'Restore object restore-backup-gcp-cs-sa is in requested state after 50 minutes.' Restore object restore-backup-gcp-cs-sa is in requested state after 50 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1