Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/logs/demand-backup-physical-azure.log grep: warning: stray \ before - Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 + '[' -n '' ']' + create_infra demand-backup-physical-azure-28654 + local ns=demand-backup-physical-azure-28654 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.RXAy8jC7Rl ++ mktemp + local LAST_ERR=/tmp/tmp.BnvJLnRgsf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RXAy8jC7Rl customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.BnvJLnRgsf + rm /tmp/tmp.RXAy8jC7Rl /tmp/tmp.BnvJLnRgsf + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.B7Lit7n1yM ++ mktemp + local LAST_ERR=/tmp/tmp.W7uNognqB0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.B7Lit7n1yM + cat /tmp/tmp.W7uNognqB0 + rm /tmp/tmp.B7Lit7n1yM /tmp/tmp.W7uNognqB0 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.5bRe2FeRPz ++ mktemp + local LAST_ERR=/tmp/tmp.v3zHu5zD1o + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5bRe2FeRPz + cat /tmp/tmp.v3zHu5zD1o + rm /tmp/tmp.5bRe2FeRPz /tmp/tmp.v3zHu5zD1o + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.XKUyfEfCIQ ++ mktemp + local LAST_ERR=/tmp/tmp.GNLV1p7slv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XKUyfEfCIQ + cat /tmp/tmp.GNLV1p7slv + rm /tmp/tmp.XKUyfEfCIQ /tmp/tmp.GNLV1p7slv + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.53mwiy4Rvz ++ mktemp + local LAST_ERR=/tmp/tmp.X3dVAoUiaS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.53mwiy4Rvz clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.X3dVAoUiaS + rm /tmp/tmp.53mwiy4Rvz /tmp/tmp.X3dVAoUiaS + return 0 + check_crd_for_deletion PR-2036-6a5cff62 + local git_tag=PR-2036-6a5cff62 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2036-6a5cff62/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0g07xP4bUv +++ mktemp ++ local LAST_ERR=/tmp/tmp.F1tzZez16F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.0g07xP4bUv ++ cat /tmp/tmp.F1tzZez16F Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.0g07xP4bUv ++ cat /tmp/tmp.F1tzZez16F Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.0g07xP4bUv ++ cat /tmp/tmp.F1tzZez16F Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.0g07xP4bUv ++ cat /tmp/tmp.F1tzZez16F Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.0g07xP4bUv /tmp/tmp.F1tzZez16F ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + awk '{print$1}' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator+ xargs kubectl delete ns ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.kveTejnLBi egrep: warning: egrep is obsolescent; using grep -E ++ mktemp + local LAST_OUT=/tmp/tmp.jaMXAWhQ3F ++ mktemp + local LAST_ERR=/tmp/tmp.VwJsHxKiGS + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.YCnCE1cVVV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kveTejnLBi + cat /tmp/tmp.VwJsHxKiGS + rm /tmp/tmp.kveTejnLBi /tmp/tmp.VwJsHxKiGS + return 0 namespace "demand-backup-physical-azure-13493" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jaMXAWhQ3F namespace "psmdb-operator" deleted + cat /tmp/tmp.YCnCE1cVVV + rm /tmp/tmp.jaMXAWhQ3F /tmp/tmp.YCnCE1cVVV + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ngGlbazwwW ++ mktemp + local LAST_ERR=/tmp/tmp.x8G7h5Mzyt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ngGlbazwwW + cat /tmp/tmp.x8G7h5Mzyt + rm /tmp/tmp.ngGlbazwwW /tmp/tmp.x8G7h5Mzyt + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.FGgIBxJY26 ++ mktemp + local LAST_ERR=/tmp/tmp.bsNhQC7clX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FGgIBxJY26 namespace/psmdb-operator created + cat /tmp/tmp.bsNhQC7clX + rm /tmp/tmp.FGgIBxJY26 /tmp/tmp.bsNhQC7clX + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.FmfsWeqmQb +++ mktemp ++ local LAST_ERR=/tmp/tmp.wDTHvCkake ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FmfsWeqmQb ++ cat /tmp/tmp.wDTHvCkake ++ rm /tmp/tmp.FmfsWeqmQb /tmp/tmp.wDTHvCkake ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.K3H6rMsUCy ++ mktemp + local LAST_ERR=/tmp/tmp.0CoLJxKPIf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.K3H6rMsUCy Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10" modified. + cat /tmp/tmp.0CoLJxKPIf + rm /tmp/tmp.K3H6rMsUCy /tmp/tmp.0CoLJxKPIf + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.86RdvWTqJT ++ mktemp + local LAST_ERR=/tmp/tmp.zAxp3O9d1F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.86RdvWTqJT customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.zAxp3O9d1F + rm /tmp/tmp.86RdvWTqJT /tmp/tmp.zAxp3O9d1F + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ARvBtcs0Dh ++ mktemp + local LAST_ERR=/tmp/tmp.mWbhIs6SRV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ARvBtcs0Dh clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.mWbhIs6SRV + rm /tmp/tmp.ARvBtcs0Dh /tmp/tmp.mWbhIs6SRV + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.sSDdh9qEwx ++ mktemp + local LAST_ERR=/tmp/tmp.n1p605Bvob + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sSDdh9qEwx deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.n1p605Bvob + rm /tmp/tmp.sSDdh9qEwx /tmp/tmp.n1p605Bvob + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.PsCAjTPemb +++ mktemp ++ local LAST_ERR=/tmp/tmp.KhwqWvx34v ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PsCAjTPemb ++ cat /tmp/tmp.KhwqWvx34v ++ rm /tmp/tmp.PsCAjTPemb /tmp/tmp.KhwqWvx34v ++ return 0 + wait_pod percona-server-mongodb-operator-dd8c7dbb4-62xjh + local pod=percona-server-mongodb-operator-dd8c7dbb4-62xjh + set +o xtrace waiting for pod/percona-server-mongodb-operator-dd8c7dbb4-62xjh to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.xHkMq38vMW +++ mktemp ++ local LAST_ERR=/tmp/tmp.VR4nEIispZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xHkMq38vMW ++ cat /tmp/tmp.VR4nEIispZ ++ rm /tmp/tmp.xHkMq38vMW /tmp/tmp.VR4nEIispZ ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-dd8c7dbb4-62xjh ++ mktemp + local LAST_OUT=/tmp/tmp.H6pB1D4COY ++ mktemp + local LAST_ERR=/tmp/tmp.eRdDxJ7lAk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-dd8c7dbb4-62xjh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H6pB1D4COY + cat /tmp/tmp.eRdDxJ7lAk + rm /tmp/tmp.H6pB1D4COY /tmp/tmp.eRdDxJ7lAk + return 0 2025-09-24T15:53:14.319Z INFO setup Manager starting up {"gitCommit": "6a5cff6212b0846db032e80907cac572ed8d3194", "gitBranch": "PR-2036-6a5cff62", "buildTime": "", "goVersion": "go1.25.1", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-azure-28654 + local namespace=demand-backup-physical-azure-28654 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-physical-azure-28654' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-azure-28654 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-azure-28654 --ignore-not-found ++ mktemp ++ mktemp egrep: warning: egrep is obsolescent; using grep -E + local LAST_OUT=/tmp/tmp.Kzsf0U5Wr8 ++ mktemp + local LAST_OUT=/tmp/tmp.OyZftgkDue + local LAST_ERR=/tmp/tmp.atDuFe8Wod + local exit_status=0 ++ mktemp + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.NPfoEfnOUP + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-physical-azure-28654 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Kzsf0U5Wr8 + cat /tmp/tmp.atDuFe8Wod + rm /tmp/tmp.Kzsf0U5Wr8 /tmp/tmp.atDuFe8Wod + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OyZftgkDue + cat /tmp/tmp.NPfoEfnOUP + rm /tmp/tmp.OyZftgkDue /tmp/tmp.NPfoEfnOUP + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-azure-28654 ++ mktemp + local LAST_OUT=/tmp/tmp.NosvJai1VG ++ mktemp + local LAST_ERR=/tmp/tmp.HiFBs9NtRm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-physical-azure-28654 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NosvJai1VG + cat /tmp/tmp.HiFBs9NtRm + rm /tmp/tmp.NosvJai1VG /tmp/tmp.HiFBs9NtRm + return 0 + desc 'create namespace demand-backup-physical-azure-28654' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-azure-28654 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-azure-28654 ++ mktemp + local LAST_OUT=/tmp/tmp.M40HHWRpZf ++ mktemp + local LAST_ERR=/tmp/tmp.f5qPZhBTTX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-physical-azure-28654 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.M40HHWRpZf namespace/demand-backup-physical-azure-28654 created + cat /tmp/tmp.f5qPZhBTTX + rm /tmp/tmp.M40HHWRpZf /tmp/tmp.f5qPZhBTTX + return 0 + set_kube_ctx demand-backup-physical-azure-28654 + local namespace=demand-backup-physical-azure-28654 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.bsFozfOQq0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dXOMeRHThm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bsFozfOQq0 ++ cat /tmp/tmp.dXOMeRHThm ++ rm /tmp/tmp.bsFozfOQq0 /tmp/tmp.dXOMeRHThm ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10 --namespace=demand-backup-physical-azure-28654 ++ mktemp + local LAST_OUT=/tmp/tmp.CK3uxFP2tc ++ mktemp + local LAST_ERR=/tmp/tmp.OjwY9YlSBO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10 --namespace=demand-backup-physical-azure-28654 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CK3uxFP2tc Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2036-6a5cff62-2-cluster10" modified. + cat /tmp/tmp.OjwY9YlSBO + rm /tmp/tmp.CK3uxFP2tc /tmp/tmp.OjwY9YlSBO + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Z1zB8wS17l ++ mktemp + local LAST_ERR=/tmp/tmp.Jj7H7M5L9M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z1zB8wS17l secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.Jj7H7M5L9M + rm /tmp/tmp.Z1zB8wS17l /tmp/tmp.Jj7H7M5L9M + return 0 + desc 'Testing on not sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on not sharded cluster ----------------------------------------------------------------------------------- + echo 'Creating PSMDB cluster' Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.uKgaO5c4aj ++ mktemp + local LAST_ERR=/tmp/tmp.vkOMcgxwBb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uKgaO5c4aj secret/some-users created + cat /tmp/tmp.vkOMcgxwBb + rm /tmp/tmp.uKgaO5c4aj /tmp/tmp.vkOMcgxwBb + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/some-name.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2036-6a5cff62"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.v9mKNyXFKB ++ mktemp + local LAST_ERR=/tmp/tmp.3L0mXZRXgA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v9mKNyXFKB perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.3L0mXZRXgA + rm /tmp/tmp.v9mKNyXFKB /tmp/tmp.3L0mXZRXgA + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.wyrQFylvjY ++ mktemp + local LAST_ERR=/tmp/tmp.iDcXZiQk2i + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wyrQFylvjY deployment.apps/psmdb-client created + cat /tmp/tmp.iDcXZiQk2i + rm /tmp/tmp.wyrQFylvjY /tmp/tmp.iDcXZiQk2i + return 0 + echo 'check if all pods started' check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jjUkSWbHrL +++ mktemp ++ local LAST_ERR=/tmp/tmp.Evq1ZyW946 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jjUkSWbHrL ++ cat /tmp/tmp.Evq1ZyW946 ++ rm /tmp/tmp.jjUkSWbHrL /tmp/tmp.Evq1ZyW946 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mthXkrotMM +++ mktemp ++ local LAST_ERR=/tmp/tmp.0qQu4yQ8Xk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mthXkrotMM ++ cat /tmp/tmp.0qQu4yQ8Xk ++ rm /tmp/tmp.mthXkrotMM /tmp/tmp.0qQu4yQ8Xk ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s4DNHaJmeW +++ mktemp ++ local LAST_ERR=/tmp/tmp.UCCcgqZZz6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s4DNHaJmeW ++ cat /tmp/tmp.UCCcgqZZz6 ++ rm /tmp/tmp.s4DNHaJmeW /tmp/tmp.UCCcgqZZz6 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L8nw5e41Dk +++ mktemp ++ local LAST_ERR=/tmp/tmp.nPdVxJ58dY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L8nw5e41Dk ++ cat /tmp/tmp.nPdVxJ58dY ++ rm /tmp/tmp.L8nw5e41Dk /tmp/tmp.nPdVxJ58dY ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish + echo 'Writing test data' Writing test data + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-azure-28654 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7zdTMnGzeO +++ mktemp ++ local LAST_ERR=/tmp/tmp.VhPjWhL2YR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7zdTMnGzeO ++ cat /tmp/tmp.VhPjWhL2YR ++ rm /tmp/tmp.7zdTMnGzeO /tmp/tmp.VhPjWhL2YR ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.dDrVyjbMKe ++ mktemp + local LAST_ERR=/tmp/tmp.HMfXrDP0PR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dDrVyjbMKe Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("22a9a877-62b3-4e84-bf16-4837acac2818") } Percona Server for MongoDB server version: v7.0.24-13 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.HMfXrDP0PR + rm /tmp/tmp.dDrVyjbMKe /tmp/tmp.HMfXrDP0PR + return 0 + sleep 1 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3EdXYZdCuz +++ mktemp ++ local LAST_ERR=/tmp/tmp.9XpJasXR4U ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3EdXYZdCuz ++ cat /tmp/tmp.9XpJasXR4U ++ rm /tmp/tmp.3EdXYZdCuz /tmp/tmp.9XpJasXR4U ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SmsOTC58XS ++ mktemp + local LAST_ERR=/tmp/tmp.2xcIIka5Vb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SmsOTC58XS Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("cba8004e-2755-4707-a79d-7fee49507bb1") } Percona Server for MongoDB server version: v7.0.24-13 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2xcIIka5Vb + rm /tmp/tmp.SmsOTC58XS /tmp/tmp.2xcIIka5Vb + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-09-24T15:56:29+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb + local suffix=.svc.cluster.local + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp egrep: warning: egrep is obsolescent; using grep -E ++ local LAST_OUT=/tmp/tmp.8CidvRY0Ca +++ mktemp ++ local LAST_ERR=/tmp/tmp.sboxtW1nu6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8CidvRY0Ca ++ cat /tmp/tmp.sboxtW1nu6 ++ rm /tmp/tmp.8CidvRY0Ca /tmp/tmp.sboxtW1nu6 ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Yag2qc9ufO ++ mktemp + local LAST_ERR=/tmp/tmp.9Ms7lvDysc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Yag2qc9ufO + cat /tmp/tmp.9Ms7lvDysc + rm /tmp/tmp.Yag2qc9ufO /tmp/tmp.9Ms7lvDysc + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/find.json /tmp/tmp.LaP0Ulr1vE/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-09-24T15:56:32+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654 mongodb '' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp egrep: warning: egrep is obsolescent; using grep -E ++ local LAST_OUT=/tmp/tmp.W5QyXg0nyG +++ mktemp ++ local LAST_ERR=/tmp/tmp.Eh3w81jggo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W5QyXg0nyG ++ cat /tmp/tmp.Eh3w81jggo ++ rm /tmp/tmp.W5QyXg0nyG /tmp/tmp.Eh3w81jggo ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.WRxFxRACQj ++ mktemp + local LAST_ERR=/tmp/tmp.NJpEvwUidV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WRxFxRACQj + cat /tmp/tmp.NJpEvwUidV + rm /tmp/tmp.WRxFxRACQj /tmp/tmp.NJpEvwUidV + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/find.json /tmp/tmp.LaP0Ulr1vE/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2025-09-24T15:56:34+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654 mongodb '' + egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp egrep: warning: egrep is obsolescent; using grep -E ++ local LAST_OUT=/tmp/tmp.TQr8JIKCSi +++ mktemp ++ local LAST_ERR=/tmp/tmp.hhb6wlPXB4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TQr8JIKCSi ++ cat /tmp/tmp.hhb6wlPXB4 ++ rm /tmp/tmp.TQr8JIKCSi /tmp/tmp.hhb6wlPXB4 ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.o7NWTkxrP2 ++ mktemp + local LAST_ERR=/tmp/tmp.CCUzEcZMey + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.o7NWTkxrP2 + cat /tmp/tmp.CCUzEcZMey + rm /tmp/tmp.o7NWTkxrP2 /tmp/tmp.CCUzEcZMey + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/find.json /tmp/tmp.LaP0Ulr1vE/find + echo 'Running AZURE backup' Running AZURE backup + backup_name_azure=backup-azure-blob + run_backup azure-blob backup-azure-blob physical + local storage=azure-blob + local backup_name=backup-azure-blob + local type=physical + desc 'run backup backup-azure-blob' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-azure-blob ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-azure-blob" | .spec.storageName = "azure-blob" | .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.lqeA3xOghz ++ mktemp + local LAST_ERR=/tmp/tmp.vkcYujkhpB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lqeA3xOghz perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.vkcYujkhpB + rm /tmp/tmp.lqeA3xOghz /tmp/tmp.vkcYujkhpB + return 0 + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state........... + check_backup_in_storage backup-azure-blob azure rs0 + local backup=backup-azure-blob + local storage_type=azure + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=engk8soperators.blob.core.windows.net ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|azure://||' ++ sed 's|s3://||' ++ local LAST_OUT=/tmp/tmp.klvfb3qxsM +++ mktemp ++ local LAST_ERR=/tmp/tmp.9J3ScOUa9Y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.klvfb3qxsM ++ cat /tmp/tmp.9J3ScOUa9Y ++ rm /tmp/tmp.klvfb3qxsM /tmp/tmp.9J3ScOUa9Y ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical/2025-09-24T15:56:38Z + [[ azure == \m\i\n\i\o ]] + local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical/2025-09-24T15:56:38Z/rs0/filelist.pbm + log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical/2025-09-24T15:56:38Z/rs0/filelist.pbm exists' + set +o xtrace [2025-09-24T15:57:01+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical/2025-09-24T15:56:38Z/rs0/filelist.pbm exists + curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical/2025-09-24T15:56:38Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 13661 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK Content-Length: 13661 Content-Type: application/octet-stream Content-MD5: jjofR5YmVQYm884Lv9fpsQ== Last-Modified: Wed, 24 Sep 2025 15:56:53 GMT ETag: 0x8DDFB82FB09D206 Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-request-id: d41c0f36-701e-0095-516b-2d37da000000 x-ms-version: 2009-09-19 x-ms-lease-status: unlocked x-ms-blob-type: BlockBlob Date: Wed, 24 Sep 2025 15:57:01 GMT + echo 'Drop collection' Drop collection + run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3we6lkXDBV +++ mktemp ++ local LAST_ERR=/tmp/tmp.QlG5K23W5w ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3we6lkXDBV ++ cat /tmp/tmp.QlG5K23W5w ++ rm /tmp/tmp.3we6lkXDBV /tmp/tmp.QlG5K23W5w ++ return 0 + local client_container=psmdb-client-b9788d8bc-zxg9k + local mongo_flag= + [[ myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SOMFZOmvG9 ++ mktemp + local LAST_ERR=/tmp/tmp.Eml0HFHG8Z + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-b9788d8bc-zxg9k -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SOMFZOmvG9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c877143d-79c3-4171-9f68-1bb90a9faad6") } Percona Server for MongoDB server version: v7.0.24-13 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.Eml0HFHG8Z + rm /tmp/tmp.SOMFZOmvG9 /tmp/tmp.Eml0HFHG8Z + return 0 + echo 'check backup and restore -- azure-blob' check backup and restore -- azure-blob + run_restore backup-azure-blob + local backup_name=backup-azure-blob + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Nqu38rGPxN ++ mktemp + local LAST_ERR=/tmp/tmp.QEySvOHUSX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Nqu38rGPxN perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created + cat /tmp/tmp.QEySvOHUSX + rm /tmp/tmp.Nqu38rGPxN /tmp/tmp.QEySvOHUSX + return 0 + run_recovery_check backup-azure-blob + local backup_name=backup-azure-blob + local compare_suffix=_restore + local is_sharded= + local tls= + wait_restore backup-azure-blob some-name requested 0 3000 + local backup_name=backup-azure-blob + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-azure-blob object to be createdOK Waiting psmdb-restore/restore-backup-azure-blob to reach state "requested" ...OK after 3 minutes + [[ 0 -eq 1 ]] + echo + compare_kubectl statefulset/some-name-rs0 _restore + local resource=statefulset/some-name-rs0 + local postfix=_restore + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/statefulset_some-name-rs0_restore.yml + local new_result=/tmp/tmp.LaP0Ulr1vE/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/statefulset_some-name-rs0_restore-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-physical-azure-28654", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.FEmGUuqgBH ++ mktemp + local LAST_ERR=/tmp/tmp.lIK6jk8Ki3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FEmGUuqgBH + cat /tmp/tmp.lIK6jk8Ki3 + rm /tmp/tmp.FEmGUuqgBH /tmp/tmp.lIK6jk8Ki3 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.LaP0Ulr1vE/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.LaP0Ulr1vE/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.LaP0Ulr1vE/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/statefulset_some-name-rs0_restore.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2036/e2e-tests/demand-backup-physical-azure/compare/statefulset_some-name-rs0_restore.yml /tmp/tmp.LaP0Ulr1vE/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2025-09-24T16:00:51+0000] compare_kubectl: statefulset/some-name-rs0 OK + wait_restore backup-azure-blob some-name ready 0 3000 + local backup_name=backup-azure-blob + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-azure-blob object to be createdOK Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" . ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.VFrobBSp9I ++ mktemp + local LAST_ERR=/tmp/tmp.iodRHHUoVz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VFrobBSp9I 2025/09/24 15:58:47 [entrypoint] starting `pbm-agent` 2025-09-24T15:59:17.000+0000 E Exit: connect to PBM: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: RSGhost, Average RTT: 4003705 }, ] } 2025/09/24 15:59:17 [entrypoint] `pbm-agent` exited with code 1 2025/09/24 15:59:17 [entrypoint] restart in 5 sec 2025/09/24 15:59:22 [entrypoint] starting `pbm-agent` 2025-09-24T15:59:52.000+0000 E Exit: connect to PBM: create mongo connection: ping: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,podName=some-name-rs0-0,serviceName=some-name,region=us-central1,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-bp5v, Average RTT: 1069414 }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, ] } 2025/09/24 15:59:52 [entrypoint] `pbm-agent` exited with code 1 2025/09/24 15:59:52 [entrypoint] restart in 5 sec 2025/09/24 15:59:57 [entrypoint] starting `pbm-agent` 2025-09-24T16:00:24.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-09-24T16:00:24.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-09-24T16:00:24.000+0000 I pbm-agent: Version: 2.10.0 Platform: linux/amd64 GitCommit: 92dfac319381e7861d6958733a3a46d2e2f7a5a1 GitBranch: release-2.10.0 BuildTime: 2025-06-23_09:47_UTC GoVersion: go1.23.8 2025-09-24T16:00:24.000+0000 I starting PITR routine 2025-09-24T16:00:24.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017 2025-09-24T16:00:24.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-09-24T16:00:24.000+0000 I listening for the commands 2025-09-24T16:00:50.000+0000 I got command restore [name: 2025-09-24T16:00:49.950573752Z, snapshot: 2025-09-24T15:56:38Z] , opid: 68d415b14c29598fc4b31251 2025-09-24T16:00:50.000+0000 I got epoch {1758729650 11} 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] backup: 2025-09-24T15:56:38Z 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] recovery started 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] port: 27419 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:51.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] mongod binary: mongod, version: v7.0.24-13 2025-09-24T16:00:51.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] restore opts: fallbackEnabled: false; allowPartlyDone: true 2025-09-24T16:00:51.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state starting 2025-09-24T16:00:51.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:51.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state starting 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] starting 2025-09-24T16:01:06.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state running 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:06.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state running 2025-09-24T16:01:21.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] send to stopAgent chan 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] stop agents heartbeats 2025-09-24T16:01:21.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] stopping mongod and flushing old data 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] shutting down server 2025-09-24T16:01:24.000+0000 D [pitr] stopping main loop 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting for the node to shutdown 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-77-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-45-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.lock 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-20-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-40-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-58-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-65-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-25-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-73-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-39-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-70-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-49-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-57-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-64-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/psmdb_telemetry.data 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-7-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-71-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-32-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-80-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/mongod.lock 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-85-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-16-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-10-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-11-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-81-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-1-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/storage.bson 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/diagnostic.data 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-4-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-6-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-72-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-83-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/sizeStorer.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-60-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-48-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-79-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-22-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.turtle 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-36-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-5-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-33-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-74-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-35-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-52-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-8-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-26-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-68-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-15-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-44-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-41-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTigerHS.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-42-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-28-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-62-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-76-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-34-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-55-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-56-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-31-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-19-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-0-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-30-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-9-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-29-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-53-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-38-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-24-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-87-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-69-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-59-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-12-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-23-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-2-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-18-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/_mdb_catalog.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-54-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-61-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-82-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/journal 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/lost+found 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-21-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-84-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-75-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/key.db 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-63-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-17-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-86-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-27-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-13-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-47-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/mongod-data 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-50-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-78-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-51-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-43-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-46-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-14-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-37-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-3-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-67-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-66-1255376988696061763.wt 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copying backup data 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerHS.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/key.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/parameters.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTiger.backup.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTiger.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/journal/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTigerHS.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/_mdb_catalog.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-0--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-10--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-12--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-14--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-18--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-19--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-2--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-22--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-24--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-27--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-29--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-32--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-34--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-36--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-39--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-4--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-42--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-43--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-46--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-49--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-52--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-54--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-57--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-6--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-60--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-62--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-65--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-68--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-72--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-74--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-78--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-8--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-80--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-82--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-84--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-86--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-1--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-11--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-13--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-15--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-20--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-21--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-23--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-25--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-26--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-28--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-3--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-30--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-31--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-33--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-35--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-37--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-38--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-40--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-41--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-44--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-45--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-47--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-48--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-5--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-50--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-51--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-53--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-55--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-56--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-58--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-59--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-61--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-63--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-64--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-66--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-67--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-69--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-7--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-70--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-71--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-73--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-75--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-76--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-77--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-79--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-81--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-83--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-85--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-87--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-9--1052562223825587613.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/sizeStorer.wt.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTiger.backup.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTiger.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/storage.bson.gz.0-114> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/journal/WiredTigerLog.0000000002.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000002.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] preparing data 2025-09-24T16:01:45.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] oplogTruncateAfterPoint: {1758729401 1} 2025-09-24T16:01:48.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] recovering oplog as standalone 2025-09-24T16:01:53.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] clean-up and reset replicaset config + cat /tmp/tmp.iodRHHUoVz Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.VFrobBSp9I /tmp/tmp.iodRHHUoVz + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.zCnngAZRPg ++ mktemp + local LAST_ERR=/tmp/tmp.GKQXSlJjXL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zCnngAZRPg 2025/09/24 15:58:13 [entrypoint] starting `pbm-agent` 2025-09-24T15:58:16.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-09-24T15:58:16.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-09-24T15:58:16.000+0000 I pbm-agent: Version: 2.10.0 Platform: linux/amd64 GitCommit: 92dfac319381e7861d6958733a3a46d2e2f7a5a1 GitBranch: release-2.10.0 BuildTime: 2025-06-23_09:47_UTC GoVersion: go1.23.8 2025-09-24T15:58:16.000+0000 I starting PITR routine 2025-09-24T15:58:16.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017 2025-09-24T15:58:16.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-09-24T15:58:16.000+0000 I listening for the commands 2025-09-24T15:59:17.000+0000 E [agentCheckup] set status: write into db: write exception: write concern error: (InterruptedDueToReplStateChange) operation was interrupted 2025/09/24 15:59:47 [ERROR] writing log: db: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: read udp 10.68.225.14:42137->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1,zone=us-central1-a, Average RTT: 2679512 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: read udp 10.68.225.14:59774->34.118.224.10:53: i/o timeout }, ] }, , entry: 2025-09-24T15:59:17.000+0000 E [agentCheckup] set status: write into db: write exception: write concern error: (InterruptedDueToReplStateChange) operation was interrupted 2025-09-24T15:59:48.000+0000 E listening commands: watch the cmd stream: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1,zone=us-central1-a, Average RTT: 3853733 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, ] } 2025/09/24 16:00:18 [ERROR] writing log: db: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-rs0-0,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-bp5v,zone=us-central1-a,region=us-central1,serviceName=some-name, Average RTT: 1651855 }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1,zone=us-central1-a, Average RTT: 1458992 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: no such host }, ] }, , entry: 2025-09-24T15:59:48.000+0000 E listening commands: watch the cmd stream: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1,zone=us-central1-a, Average RTT: 3853733 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, ] } 2025-09-24T16:00:17.000+0000 E [agentCheckup] detecting parallel agent: get status: query mongo: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: read udp 10.68.225.14:45349->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1, Average RTT: 1651459 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: no such host }, ] } 2025-09-24T16:00:24.000+0000 E listening commands: watch the cmd stream: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: read udp 10.68.225.14:42137->34.118.224.10:53: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: zone=us-central1-a,podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz,region=us-central1, Average RTT: 2254157 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local on 34.118.224.10:53: read udp 10.68.225.14:59774->34.118.224.10:53: i/o timeout }, ] } 2025-09-24T16:00:02.000+0000 E [pitr] init: get conf: get: server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, { Addr: some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: RSSecondary, Tag sets: region=us-central1,zone=us-central1-a,podName=some-name-rs0-1,serviceName=some-name,nodeName=gke-jen-psmdb-2036-6a5cf-default-pool-91e75855-j5cz, Average RTT: 3195741 }, { Addr: some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local: i/o timeout }, ] } 2025-09-24T16:00:50.000+0000 I got command restore [name: 2025-09-24T16:00:49.950573752Z, snapshot: 2025-09-24T15:56:38Z] , opid: 68d415b14c29598fc4b31251 2025-09-24T16:00:50.000+0000 I got epoch {1758729427 1} 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] oplog slicer disabled 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] backup: 2025-09-24T15:56:38Z 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] recovery started 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] port: 27325 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] mongod binary: mongod, version: v7.0.24-13 2025-09-24T16:00:51.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] restore opts: fallbackEnabled: false; allowPartlyDone: true 2025-09-24T16:00:51.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state starting 2025-09-24T16:00:51.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:51.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for `starting` status in rs map[.pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{}] 2025-09-24T16:00:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:56.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for shards map[.pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/rs:{}] 2025-09-24T16:01:01.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:01.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state starting 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] starting 2025-09-24T16:01:06.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state running 2025-09-24T16:01:06.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:06.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for `running` status in rs map[.pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{}] 2025-09-24T16:01:12.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:12.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for shards map[.pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/rs:{}] 2025-09-24T16:01:17.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:17.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:22.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state running 2025-09-24T16:01:22.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] send to stopAgent chan 2025-09-24T16:01:22.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] stop agents heartbeats 2025-09-24T16:01:22.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] stopping mongod and flushing old data 2025-09-24T16:01:22.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] shutting down server 2025-09-24T16:01:22.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:23.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:24.000+0000 D [pitr] stopping main loop 2025-09-24T16:01:24.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:25.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:26.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:27.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:28.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:29.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:30.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:31.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:32.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:33.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:34.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:35.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:36.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:38.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:39.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:40.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:41.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:42.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:43.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:44.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down 2025-09-24T16:01:45.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting to become secondary before shutting it down + cat /tmp/tmp.GKQXSlJjXL Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.zCnngAZRPg /tmp/tmp.GKQXSlJjXL + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.rAI3xKWRWE ++ mktemp + local LAST_ERR=/tmp/tmp.JWlmWGqZ9I + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rAI3xKWRWE 2025/09/24 16:00:36 [entrypoint] starting `pbm-agent` 2025-09-24T16:00:40.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2025-09-24T16:00:40.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2025-09-24T16:00:40.000+0000 I pbm-agent: Version: 2.10.0 Platform: linux/amd64 GitCommit: 92dfac319381e7861d6958733a3a46d2e2f7a5a1 GitBranch: release-2.10.0 BuildTime: 2025-06-23_09:47_UTC GoVersion: go1.23.8 2025-09-24T16:00:40.000+0000 I starting PITR routine 2025-09-24T16:00:40.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017 2025-09-24T16:00:40.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2025-09-24T16:00:40.000+0000 I listening for the commands 2025-09-24T16:00:50.000+0000 I got command restore [name: 2025-09-24T16:00:49.950573752Z, snapshot: 2025-09-24T15:56:38Z] , opid: 68d415b14c29598fc4b31251 2025-09-24T16:00:50.000+0000 I got epoch {1758729427 1} 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] backup: 2025-09-24T15:56:38Z 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] recovery started 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] port: 27079 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] mongod binary: mongod, version: v7.0.24-13 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] restore opts: fallbackEnabled: false; allowPartlyDone: true 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state starting 2025-09-24T16:00:50.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:00:50.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:05.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state starting 2025-09-24T16:01:05.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] starting 2025-09-24T16:01:05.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state running 2025-09-24T16:01:05.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:06.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for cluster 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] converged to state running 2025-09-24T16:01:21.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] send to stopAgent chan 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] stop agents heartbeats 2025-09-24T16:01:21.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] stopping mongod and flushing old data 2025-09-24T16:01:21.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] shutting down server 2025-09-24T16:01:25.000+0000 D [pitr] stopping main loop 2025-09-24T16:01:36.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] waiting for the node to shutdown 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-61--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.turtle 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-53--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-7--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-9--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-28--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-73--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-70--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-22--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/_mdb_catalog.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-69--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-59--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-11--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-23--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-27--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-82--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-84--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-58--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/diagnostic.data 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-72--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-37--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-71--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-29--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-52--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-62--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-44--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-51--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-46--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-57--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-30--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-79--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-4--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-55--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-1--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-10--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-35--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-47--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-76--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-85--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-19--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-24--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-40--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-31--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-75--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-18--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-26--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-41--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/key.db 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-21--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-68--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/sizeStorer.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-34--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-49--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/storage.bson 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-87--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-83--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-74--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-20--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-14--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-65--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-45--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-36--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-39--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-77--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-54--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-25--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/lost+found 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/journal 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-86--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-56--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-48--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-64--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-80--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-33--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-42--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-60--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger.lock 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-2--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-0--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTigerHS.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-43--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-5--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-38--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-3--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/mongod.lock 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/psmdb_telemetry.data 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-78--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-67--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-81--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-12--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-63--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-66--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/WiredTiger 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-32--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-13--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-6--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/mongod-data 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-50--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/index-15--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] remove /data/db/collection-8--1683679287013441829.wt 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copying backup data 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerHS.wt.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/key.wt.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/parameters.wt.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTiger.backup.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTiger.gz> to 2025-09-24T16:01:37.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/journal/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTigerHS.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/_mdb_catalog.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-0--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-10--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-12--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-14--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-18--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-19--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-2--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-22--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-24--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-27--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-29--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-32--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-34--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-36--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-39--1052562223825587613.wt.gz> to 2025-09-24T16:01:38.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-4--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-42--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-43--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-46--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-49--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-52--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-54--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-57--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-6--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-60--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-62--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-65--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-68--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-72--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-74--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-78--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-8--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-80--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-82--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-84--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/collection-86--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-1--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-11--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-13--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-15--1052562223825587613.wt.gz> to 2025-09-24T16:01:39.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-20--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-21--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-23--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-25--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-26--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-28--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-3--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-30--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-31--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-33--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-35--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-37--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-38--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-40--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-41--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-44--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-45--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-47--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-48--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-5--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-50--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-51--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-53--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-55--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-56--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-58--1052562223825587613.wt.gz> to 2025-09-24T16:01:40.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-59--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-61--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-63--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-64--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-66--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-67--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-69--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-7--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-70--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-71--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-73--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-75--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-76--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-77--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-79--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-81--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-83--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-85--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-87--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/index-9--1052562223825587613.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/sizeStorer.wt.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTiger.backup.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/WiredTiger.gz> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/storage.bson.gz.0-114> to 2025-09-24T16:01:41.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/journal/WiredTigerLog.0000000002.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000001.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] copy <2025-09-24T15:56:38Z/rs0/key.db/WiredTigerLog.0000000002.gz> to 2025-09-24T16:01:42.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] preparing data 2025-09-24T16:01:45.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] oplogTruncateAfterPoint: {1758729401 1} 2025-09-24T16:01:48.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] recovering oplog as standalone 2025-09-24T16:01:53.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] clean-up and reset replicaset config 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmAgents' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmPITRChunks' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmRestores' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmPITR' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmOpLog' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmCmd' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmLock' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmLog' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmLock' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmLockOp' 2025-09-24T16:01:56.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] dropping 'admin.pbmBackups' 2025-09-24T16:01:58.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] restore on node succeed 2025-09-24T16:01:58.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] moving to state done 2025-09-24T16:01:58.000+0000 D [restore/2025-09-24T16:00:49.950573752Z] BufferSize is set to 10485760 (~10Mb) | 10 2025-09-24T16:01:58.000+0000 I [restore/2025-09-24T16:00:49.950573752Z] waiting for `done` status in rs map[.pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{} .pbm.restore/2025-09-24T16:00:49.950573752Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-azure-28654.svc.cluster.local:27017:{}] + cat /tmp/tmp.JWlmWGqZ9I Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.rAI3xKWRWE /tmp/tmp.JWlmWGqZ9I + return 0 + grep restore-backup-azure-blob + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.mSoxoDlk0S +++ mktemp ++ local LAST_ERR=/tmp/tmp.iybssTC2xp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mSoxoDlk0S ++ cat /tmp/tmp.iybssTC2xp ++ rm /tmp/tmp.mSoxoDlk0S /tmp/tmp.iybssTC2xp ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-dd8c7dbb4-62xjh ++ mktemp + local LAST_OUT=/tmp/tmp.rbtKxV4ySd ++ mktemp + local LAST_ERR=/tmp/tmp.IVbmNJvuDv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-dd8c7dbb4-62xjh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rbtKxV4ySd + cat /tmp/tmp.IVbmNJvuDv + rm /tmp/tmp.rbtKxV4ySd /tmp/tmp.IVbmNJvuDv + return 0 2025-09-24T15:57:38.538Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "c09d4380-c781-4e5f-9986-25e22c897818", "ready": false} 2025-09-24T15:57:43.539Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "c3eadf6b-99fd-47fe-b9a0-fc6f29ddb652", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:57:43.540Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "c3eadf6b-99fd-47fe-b9a0-fc6f29ddb652", "ready": false} 2025-09-24T15:57:48.541Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "87e6d75e-70d6-4cd3-9202-57be5f19bc90", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:57:48.542Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "87e6d75e-70d6-4cd3-9202-57be5f19bc90", "ready": false} 2025-09-24T15:57:53.542Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "80bf1769-2e68-42bd-8afb-35e4e66b9400", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:57:53.543Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "80bf1769-2e68-42bd-8afb-35e4e66b9400", "ready": false} 2025-09-24T15:57:58.545Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "4852a220-00c0-48f8-9b66-caef44ee1e32", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:57:58.545Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "4852a220-00c0-48f8-9b66-caef44ee1e32", "ready": false} 2025-09-24T15:58:03.546Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "dfa6d4aa-1200-4a57-bf39-f5014d784228", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:03.547Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "dfa6d4aa-1200-4a57-bf39-f5014d784228", "ready": false} 2025-09-24T15:58:08.547Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "a0fe663c-f329-44b7-a0df-cf91d983cc25", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:08.548Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "a0fe663c-f329-44b7-a0df-cf91d983cc25", "ready": false} 2025-09-24T15:58:13.549Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "72cc3d4e-bdd7-4b49-ae09-ffd101d291c7", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:13.550Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "72cc3d4e-bdd7-4b49-ae09-ffd101d291c7", "ready": false} 2025-09-24T15:58:18.550Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "ec5075b0-794a-4e1c-8908-78f8d6ba443e", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:18.551Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "ec5075b0-794a-4e1c-8908-78f8d6ba443e", "ready": false} 2025-09-24T15:58:23.551Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d4d65b5e-748c-4d2d-8299-11f959bb1c83", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:23.552Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d4d65b5e-748c-4d2d-8299-11f959bb1c83", "ready": false} 2025-09-24T15:58:28.553Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "a1f4289f-17f5-4cc9-a4af-f3dc44e83044", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:28.554Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "a1f4289f-17f5-4cc9-a4af-f3dc44e83044", "ready": false} 2025-09-24T15:58:33.554Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "05b2463d-eae5-4303-817c-62b1785c9478", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:33.555Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "05b2463d-eae5-4303-817c-62b1785c9478", "ready": false} 2025-09-24T15:58:38.555Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "9e131bbd-720a-4ff8-9b20-e928af68f96c", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:38.556Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "9e131bbd-720a-4ff8-9b20-e928af68f96c", "ready": false} 2025-09-24T15:58:43.557Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7d53b271-c6e1-4638-b3ba-0f055351da2c", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:43.558Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7d53b271-c6e1-4638-b3ba-0f055351da2c", "ready": false} 2025-09-24T15:58:48.560Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "54cba9e0-bbac-4476-815b-79cccdc002bf", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:48.560Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "54cba9e0-bbac-4476-815b-79cccdc002bf", "ready": false} 2025-09-24T15:58:53.560Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "5de2507c-db07-46ed-b41c-56c9c52a3974", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:53.561Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "5de2507c-db07-46ed-b41c-56c9c52a3974", "ready": false} 2025-09-24T15:58:58.562Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "2d509aca-7d1d-4a92-8f2c-1e2a25db3ef7", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:58:58.563Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "2d509aca-7d1d-4a92-8f2c-1e2a25db3ef7", "ready": false} 2025-09-24T15:59:03.564Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d2ca243b-ebff-46d1-b08a-5b28bf49e64d", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:03.564Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d2ca243b-ebff-46d1-b08a-5b28bf49e64d", "ready": false} 2025-09-24T15:59:08.565Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "fc44167d-7e2a-4c4f-ba04-d3ac12fbf1e3", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:08.566Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "fc44167d-7e2a-4c4f-ba04-d3ac12fbf1e3", "ready": false} 2025-09-24T15:59:13.567Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8fcba315-97ed-4611-abf7-a43bfe7816c3", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:13.568Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8fcba315-97ed-4611-abf7-a43bfe7816c3", "ready": false} 2025-09-24T15:59:18.569Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7af52a34-f27e-41aa-bf1f-01e3dba9269c", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:18.569Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7af52a34-f27e-41aa-bf1f-01e3dba9269c", "ready": false} 2025-09-24T15:59:23.570Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "13969c23-81bd-4c2a-96ac-afdffc03bd3c", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:23.571Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "13969c23-81bd-4c2a-96ac-afdffc03bd3c", "ready": false} 2025-09-24T15:59:28.572Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "48f3d96f-7e21-4439-a835-f4322dc7eba2", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:28.573Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "48f3d96f-7e21-4439-a835-f4322dc7eba2", "ready": false} 2025-09-24T15:59:33.574Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "1c86ca00-1bc5-4015-8814-6e17c2deb53b", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:33.574Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "1c86ca00-1bc5-4015-8814-6e17c2deb53b", "ready": false} 2025-09-24T15:59:38.575Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d6cce73c-bc78-4353-ac3a-9b2a3fea3f4c", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:38.576Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d6cce73c-bc78-4353-ac3a-9b2a3fea3f4c", "ready": false} 2025-09-24T15:59:43.577Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "6e5b0815-92f7-4b20-9687-90a5c60fe242", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:43.578Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "6e5b0815-92f7-4b20-9687-90a5c60fe242", "ready": false} 2025-09-24T15:59:48.579Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d24bcb1a-a91b-4ba3-99ea-0fc3b9ce54e4", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:48.580Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "d24bcb1a-a91b-4ba3-99ea-0fc3b9ce54e4", "ready": false} 2025-09-24T15:59:53.581Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "4b57cebe-eef2-46db-bfad-6693f55ea9e7", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:53.582Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "4b57cebe-eef2-46db-bfad-6693f55ea9e7", "ready": false} 2025-09-24T15:59:58.583Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8974f7c7-9d53-42df-971c-4883192dc533", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T15:59:58.583Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8974f7c7-9d53-42df-971c-4883192dc533", "ready": false} 2025-09-24T16:00:03.584Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "ff610b83-5dce-419e-a2b2-61f3ec6fba16", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:03.585Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "ff610b83-5dce-419e-a2b2-61f3ec6fba16", "ready": false} 2025-09-24T16:00:08.586Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "3501d614-14a7-4f98-b46e-ccd52f952653", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:08.587Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "3501d614-14a7-4f98-b46e-ccd52f952653", "ready": false} 2025-09-24T16:00:13.588Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "24006166-8844-49e4-bcd0-f9e55c743266", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:13.589Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "24006166-8844-49e4-bcd0-f9e55c743266", "ready": false} 2025-09-24T16:00:18.590Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8028c7ae-6967-4184-953c-f1dd813f6ccf", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:18.590Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "8028c7ae-6967-4184-953c-f1dd813f6ccf", "ready": false} 2025-09-24T16:00:23.591Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "9be564bf-92df-4820-b7fe-be5e0cec7c28", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:23.592Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "9be564bf-92df-4820-b7fe-be5e0cec7c28", "ready": false} 2025-09-24T16:00:28.593Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "24b1136e-af29-47d7-9bf3-ff91c603f8d8", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:28.594Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "24b1136e-af29-47d7-9bf3-ff91c603f8d8", "ready": false} 2025-09-24T16:00:33.595Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "05f1be06-026f-4fd3-9f7b-820673d00a03", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:33.595Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "05f1be06-026f-4fd3-9f7b-820673d00a03", "ready": false} 2025-09-24T16:00:38.597Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "5e69a6d7-f4cb-4d62-a7e2-0c1d1e2e2a78", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:38.597Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "5e69a6d7-f4cb-4d62-a7e2-0c1d1e2e2a78", "ready": false} 2025-09-24T16:00:43.598Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "907dc349-1955-4fa9-87a8-a2d8d732f0f8", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:43.599Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "907dc349-1955-4fa9-87a8-a2d8d732f0f8", "ready": false} 2025-09-24T16:00:48.599Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:48.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "pod": "some-name-rs0-2"} 2025-09-24T16:00:48.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "pod": "some-name-rs0-0"} 2025-09-24T16:00:48.600Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "pod": "some-name-rs0-1"} 2025-09-24T16:00:48.600Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "sts": "some-name-rs0", "replset": "rs0"} 2025-09-24T16:00:49.377Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "command": ["/opt/percona/pbm", "restore", "2025-09-24T15:56:38Z", "--out", "json"], "pod": "some-name-rs0-0"} 2025-09-24T16:00:50.031Z INFO Restore started {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "pod": "some-name-rs0-0"} 2025-09-24T16:00:50.031Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "7a61f72b-8181-4187-8242-6c8d9fbca493", "previous": "waiting", "current": "requested"} 2025-09-24T16:00:50.625Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:50.626Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "pod": "some-name-rs0-0"} 2025-09-24T16:00:50.626Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "pod": "some-name-rs0-1"} 2025-09-24T16:00:50.626Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "pod": "some-name-rs0-2"} 2025-09-24T16:00:50.626Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "sts": "some-name-rs0", "replset": "rs0"} 2025-09-24T16:00:50.626Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-24T16:00:49.950573752Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-09-24T16:00:51.861Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "764388a8-2230-4f27-9c38-1767d328f8f7", "status": {"type":"physical","opid":"","name":"2025-09-24T16:00:49.950573752Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2025-09-24T16:00:55.626Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "cluster": "some-name", "namespace": "demand-backup-physical-azure-28654"} 2025-09-24T16:00:55.627Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "pod": "some-name-rs0-2"} 2025-09-24T16:00:55.627Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "pod": "some-name-rs0-0"} 2025-09-24T16:00:55.627Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "pod": "some-name-rs0-1"} 2025-09-24T16:00:55.627Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "sts": "some-name-rs0", "replset": "rs0"} 2025-09-24T16:00:55.627Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "command": ["/opt/percona/pbm", "describe-restore", "2025-09-24T16:00:49.950573752Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2025-09-24T16:00:56.223Z ERROR failed to make restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "restore": "restore-backup-azure-blob", "backup": "backup-azure-blob", "error": "unmarshal PBM describe-restore output: unexpected end of JSON input", "errorVerbose": "unexpected end of JSON input\nunmarshal PBM describe-restore output\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcilePhysicalRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:225\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:250\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} 2025-09-24T16:00:56.223Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "previous": "requested", "current": "error"} 2025-09-24T16:00:56.820Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e"} 2025-09-24T16:00:56.820Z ERROR Reconciler error {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"}, "namespace": "demand-backup-physical-azure-28654", "name": "restore-backup-azure-blob", "reconcileID": "bc6a3f63-402a-4eec-b3a5-97bf1edfb91e", "error": "reconcile physical restore: unmarshal PBM describe-restore output: unexpected end of JSON input", "errorVerbose": "unexpected end of JSON input\nunmarshal PBM describe-restore output\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).reconcilePhysicalRestore\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/physical.go:225\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:250\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile physical restore\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore.(*ReconcilePerconaServerMongoDBRestore).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbrestore/psmdb_restore_controller.go:252\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"} + kubectl_bin get psmdb-restore restore-backup-azure-blob -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.UeW6tC3A5n ++ mktemp + local LAST_ERR=/tmp/tmp.EfYM3pzAoY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-azure-blob -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UeW6tC3A5n apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-azure-blob","namespace":"demand-backup-physical-azure-28654"},"spec":{"backupName":"backup-azure-blob","clusterName":"some-name"}} creationTimestamp: "2025-09-24T15:57:06Z" generation: 1 name: restore-backup-azure-blob namespace: demand-backup-physical-azure-28654 resourceVersion: "1758729656255471006" uid: 7f83de24-e3d2-4103-8570-12b17c58485d spec: backupName: backup-azure-blob clusterName: some-name status: error: 'unmarshal PBM describe-restore output: unexpected end of JSON input' pbmName: "2025-09-24T16:00:49.950573752Z" state: error + cat /tmp/tmp.EfYM3pzAoY + rm /tmp/tmp.UeW6tC3A5n /tmp/tmp.EfYM3pzAoY + return 0 + kubectl_bin describe psmdb-restore restore-backup-azure-blob ++ mktemp + local LAST_OUT=/tmp/tmp.QtpHxyISvG ++ mktemp + local LAST_ERR=/tmp/tmp.eQ2xPnLg5d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-azure-blob + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QtpHxyISvG Name: restore-backup-azure-blob Namespace: demand-backup-physical-azure-28654 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2025-09-24T15:57:06Z Generation: 1 Resource Version: 1758729656255471006 UID: 7f83de24-e3d2-4103-8570-12b17c58485d Spec: Backup Name: backup-azure-blob Cluster Name: some-name Status: Error: unmarshal PBM describe-restore output: unexpected end of JSON input Pbm Name: 2025-09-24T16:00:49.950573752Z State: error Events: + cat /tmp/tmp.eQ2xPnLg5d + rm /tmp/tmp.QtpHxyISvG /tmp/tmp.eQ2xPnLg5d + return 0 + echo 'Restore object restore-backup-azure-blob is in error state after 1 minutes.' Restore object restore-backup-azure-blob is in error state after 1 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1