Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/logs/pitr-physical.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + main + create_infra pitr-physical-15995 + local ns=pitr-physical-15995 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.q9qC8FhNcL ++ mktemp + local LAST_ERR=/tmp/tmp.loU5d0Zrk3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q9qC8FhNcL customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.loU5d0Zrk3 + rm /tmp/tmp.q9qC8FhNcL /tmp/tmp.loU5d0Zrk3 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.drj9Ad149U ++ mktemp + local LAST_ERR=/tmp/tmp.g2nCqniv3x + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.drj9Ad149U + cat /tmp/tmp.g2nCqniv3x + rm /tmp/tmp.drj9Ad149U /tmp/tmp.g2nCqniv3x + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.VmTKgYUDqf ++ mktemp + local LAST_ERR=/tmp/tmp.fV3JsKIm2F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VmTKgYUDqf + cat /tmp/tmp.fV3JsKIm2F + rm /tmp/tmp.VmTKgYUDqf /tmp/tmp.fV3JsKIm2F + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ZRycr3vFwf ++ mktemp + local LAST_ERR=/tmp/tmp.Gn0wjFInin + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZRycr3vFwf + cat /tmp/tmp.Gn0wjFInin + rm /tmp/tmp.ZRycr3vFwf /tmp/tmp.Gn0wjFInin + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.5IO92FL3xX ++ mktemp + local LAST_ERR=/tmp/tmp.KUr7rJNU3y + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5IO92FL3xX clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted serviceaccount "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.KUr7rJNU3y + rm /tmp/tmp.5IO92FL3xX /tmp/tmp.KUr7rJNU3y + return 0 + check_crd_for_deletion PR-1915-2d829121 + local git_tag=PR-1915-2d829121 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1915-2d829121/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vMAbNOsDLz +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ed6tnNScEp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.vMAbNOsDLz ++ cat /tmp/tmp.Ed6tnNScEp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.vMAbNOsDLz ++ cat /tmp/tmp.Ed6tnNScEp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.vMAbNOsDLz ++ cat /tmp/tmp.Ed6tnNScEp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.vMAbNOsDLz ++ cat /tmp/tmp.Ed6tnNScEp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.vMAbNOsDLz /tmp/tmp.Ed6tnNScEp ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.73obAoo78T + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + xargs kubectl delete ns + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.kx3YmhIeDH ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.Cft8Ds0owl + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.UnzCAblJXd + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kx3YmhIeDH + cat /tmp/tmp.UnzCAblJXd + rm /tmp/tmp.kx3YmhIeDH /tmp/tmp.UnzCAblJXd + return 0 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.73obAoo78T namespace "psmdb-operator" deleted + cat /tmp/tmp.Cft8Ds0owl + rm /tmp/tmp.73obAoo78T /tmp/tmp.Cft8Ds0owl + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.EtjJ4VQbPd ++ mktemp + local LAST_ERR=/tmp/tmp.u4psoK3Cea + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EtjJ4VQbPd + cat /tmp/tmp.u4psoK3Cea + rm /tmp/tmp.EtjJ4VQbPd /tmp/tmp.u4psoK3Cea + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.umz9EbpD8b ++ mktemp + local LAST_ERR=/tmp/tmp.2jQ6zM1UVC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.umz9EbpD8b namespace/psmdb-operator created + cat /tmp/tmp.2jQ6zM1UVC + rm /tmp/tmp.umz9EbpD8b /tmp/tmp.2jQ6zM1UVC + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.REDwyeiels +++ mktemp ++ local LAST_ERR=/tmp/tmp.CmcX2Zz5qH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.REDwyeiels ++ cat /tmp/tmp.CmcX2Zz5qH ++ rm /tmp/tmp.REDwyeiels /tmp/tmp.CmcX2Zz5qH ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.CNg9KnYhKZ ++ mktemp + local LAST_ERR=/tmp/tmp.4wqp5840JA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CNg9KnYhKZ Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2" modified. + cat /tmp/tmp.4wqp5840JA + rm /tmp/tmp.CNg9KnYhKZ /tmp/tmp.4wqp5840JA + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.WnUcVoVrXa ++ mktemp + local LAST_ERR=/tmp/tmp.TrqlgqDGRZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WnUcVoVrXa customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.TrqlgqDGRZ + rm /tmp/tmp.WnUcVoVrXa /tmp/tmp.TrqlgqDGRZ + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.FA0HKWfiCE ++ mktemp + local LAST_ERR=/tmp/tmp.i8XehzzhzW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FA0HKWfiCE clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.i8XehzzhzW + rm /tmp/tmp.FA0HKWfiCE /tmp/tmp.i8XehzzhzW + return 0 + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.g9gegb4sRN ++ mktemp + local LAST_ERR=/tmp/tmp.SMS0t7ffHY + local exit_status=0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | + local timeout=4 ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-operator.yaml ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.g9gegb4sRN deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.SMS0t7ffHY + rm /tmp/tmp.g9gegb4sRN /tmp/tmp.SMS0t7ffHY + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.fJncKNxOpi +++ mktemp ++ local LAST_ERR=/tmp/tmp.OxrPHi4tcF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fJncKNxOpi ++ cat /tmp/tmp.OxrPHi4tcF ++ rm /tmp/tmp.fJncKNxOpi /tmp/tmp.OxrPHi4tcF ++ return 0 + wait_pod percona-server-mongodb-operator-6c96ccdc79-z5mdz + local pod=percona-server-mongodb-operator-6c96ccdc79-z5mdz + set +o xtrace waiting for pod/percona-server-mongodb-operator-6c96ccdc79-z5mdz to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.vavF3CE4Og +++ mktemp ++ local LAST_ERR=/tmp/tmp.vxayLmzJvj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vavF3CE4Og ++ cat /tmp/tmp.vxayLmzJvj ++ rm /tmp/tmp.vavF3CE4Og /tmp/tmp.vxayLmzJvj ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6c96ccdc79-z5mdz ++ mktemp + local LAST_OUT=/tmp/tmp.ZrStS0N09Z ++ mktemp + local LAST_ERR=/tmp/tmp.NctwBI568F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-6c96ccdc79-z5mdz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZrStS0N09Z + cat /tmp/tmp.NctwBI568F + rm /tmp/tmp.ZrStS0N09Z /tmp/tmp.NctwBI568F + return 0 2025-05-21T10:30:18.803Z INFO setup Manager starting up {"gitCommit": "2d8291219bbb1a7ad015786c5ebd76d37f8fba63", "gitBranch": "PR-1915-2d829121", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} + create_namespace pitr-physical-15995 + local namespace=pitr-physical-15995 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-physical-15995' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-physical-15995 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-physical-15995 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.wtB5K99bss + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.1J9ZLGNPy4 + local exit_status=0 + local timeout=4 + local LAST_OUT=/tmp/tmp.s5JaTbQMer + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + awk '{print$1}' ++ mktemp + local LAST_ERR=/tmp/tmp.mu5aHnAghW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pitr-physical-15995 --ignore-not-found ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wtB5K99bss + cat /tmp/tmp.1J9ZLGNPy4 + rm /tmp/tmp.wtB5K99bss /tmp/tmp.1J9ZLGNPy4 + return 0 + kubectl_bin wait --for=delete namespace pitr-physical-15995 ++ mktemp + local LAST_OUT=/tmp/tmp.qbGE8N30yH ++ mktemp + local LAST_ERR=/tmp/tmp.y0IyNzVMw6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace pitr-physical-15995 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s5JaTbQMer + cat /tmp/tmp.mu5aHnAghW + rm /tmp/tmp.s5JaTbQMer /tmp/tmp.mu5aHnAghW + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qbGE8N30yH + cat /tmp/tmp.y0IyNzVMw6 + rm /tmp/tmp.qbGE8N30yH /tmp/tmp.y0IyNzVMw6 + return 0 + desc 'create namespace pitr-physical-15995' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-physical-15995 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-physical-15995 ++ mktemp + local LAST_OUT=/tmp/tmp.D484wRyoHG ++ mktemp + local LAST_ERR=/tmp/tmp.XY39AYC1sX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pitr-physical-15995 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D484wRyoHG namespace/pitr-physical-15995 created + cat /tmp/tmp.XY39AYC1sX + rm /tmp/tmp.D484wRyoHG /tmp/tmp.XY39AYC1sX + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZFt7L8hug3 namespace "gmp-public" deleted +++ mktemp ++ local LAST_ERR=/tmp/tmp.8btXUwk0cN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context namespace "gmp-system" deleted ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZFt7L8hug3 ++ cat /tmp/tmp.8btXUwk0cN ++ rm /tmp/tmp.ZFt7L8hug3 /tmp/tmp.8btXUwk0cN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2 --namespace=pitr-physical-15995 ++ mktemp + local LAST_OUT=/tmp/tmp.mk7SroLODo ++ mktemp + local LAST_ERR=/tmp/tmp.TOC5X8Ky0c + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2 --namespace=pitr-physical-15995 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mk7SroLODo Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster2" modified. + cat /tmp/tmp.TOC5X8Ky0c + rm /tmp/tmp.mk7SroLODo /tmp/tmp.TOC5X8Ky0c + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Wed May 21 10:30:59 2025 NAMESPACE: pitr-physical-15995 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-physical-15995.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-15995 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-15995 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-15995 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-15995 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K9oI47QXJs +++ mktemp ++ local LAST_ERR=/tmp/tmp.HrQ81o01R8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K9oI47QXJs ++ cat /tmp/tmp.HrQ81o01R8 ++ rm /tmp/tmp.K9oI47QXJs /tmp/tmp.HrQ81o01R8 ++ return 0 + MINIO_POD=minio-service-86dfccd949-22gjk + wait_pod minio-service-86dfccd949-22gjk + local pod=minio-service-86dfccd949-22gjk + set +o xtrace waiting for pod/minio-service-86dfccd949-22gjk to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-15995.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.ru9QyvJW72 ++ mktemp + local LAST_ERR=/tmp/tmp.FHMKEv0xAK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-15995.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ru9QyvJW72 service/minio-service created + cat /tmp/tmp.FHMKEv0xAK + rm /tmp/tmp.ru9QyvJW72 /tmp/tmp.FHMKEv0xAK + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.8eZEDeJ1Zg ++ mktemp + local LAST_ERR=/tmp/tmp.uNHY8EeAMF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8eZEDeJ1Zg make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.uNHY8EeAMF + rm /tmp/tmp.8eZEDeJ1Zg /tmp/tmp.uNHY8EeAMF + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0fLFxG2RjN ++ mktemp + local LAST_ERR=/tmp/tmp.ejY46SaBLS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0fLFxG2RjN secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.ejY46SaBLS + rm /tmp/tmp.0fLFxG2RjN /tmp/tmp.ejY46SaBLS + return 0 + cluster=some-name + desc 'create first PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/conf/some-name-rs0.yml + kubectl_bin apply -f - ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121"' + local LAST_OUT=/tmp/tmp.0r8ZGz8LTr + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' ++ mktemp + local LAST_ERR=/tmp/tmp.x4l1yERZU4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '.spec.upgradeOptions.apply="Never"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0r8ZGz8LTr perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.x4l1yERZU4 + rm /tmp/tmp.0r8ZGz8LTr /tmp/tmp.x4l1yERZU4 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nwOhFI2fCh +++ mktemp ++ local LAST_ERR=/tmp/tmp.hktpfPwj0S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nwOhFI2fCh ++ cat /tmp/tmp.hktpfPwj0S ++ rm /tmp/tmp.nwOhFI2fCh /tmp/tmp.hktpfPwj0S ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WGUGJG2fOu +++ mktemp ++ local LAST_ERR=/tmp/tmp.mpFVq43K26 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WGUGJG2fOu ++ cat /tmp/tmp.mpFVq43K26 ++ rm /tmp/tmp.WGUGJG2fOu /tmp/tmp.mpFVq43K26 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness......................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.omkHMlTTeY +++ mktemp ++ local LAST_ERR=/tmp/tmp.wtuY7tQhsz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.omkHMlTTeY ++ cat /tmp/tmp.wtuY7tQhsz ++ rm /tmp/tmp.omkHMlTTeY /tmp/tmp.wtuY7tQhsz ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1gwKAhbPyS +++ mktemp ++ local LAST_ERR=/tmp/tmp.JMrJwE3sKF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1gwKAhbPyS ++ cat /tmp/tmp.JMrJwE3sKF ++ rm /tmp/tmp.1gwKAhbPyS /tmp/tmp.JMrJwE3sKF ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + sleep 10 + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-15995 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-15995 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hcLgpS5FRS +++ mktemp ++ local LAST_ERR=/tmp/tmp.YCC5f7JpcL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hcLgpS5FRS ++ cat /tmp/tmp.YCC5f7JpcL ++ rm /tmp/tmp.hcLgpS5FRS /tmp/tmp.YCC5f7JpcL ++ return 0 + local client_container=psmdb-client-66f577db5f-lp7s6 + kubectl_bin exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.w1ovxRddal ++ mktemp + local LAST_ERR=/tmp/tmp.NkydNVwbug + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.w1ovxRddal Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("050dec70-6a1a-46d5-bf18-fb438757a5c0") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.NkydNVwbug + rm /tmp/tmp.w1ovxRddal /tmp/tmp.NkydNVwbug + return 0 + sleep 2 + write_document + local cmp_postfix= + local sleep_value=0 + desc 'write initial data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write initial data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-15995 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-15995 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hN517sZ897 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ou4iEyE2HF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hN517sZ897 ++ cat /tmp/tmp.ou4iEyE2HF ++ rm /tmp/tmp.hN517sZ897 /tmp/tmp.ou4iEyE2HF ++ return 0 + local client_container=psmdb-client-66f577db5f-lp7s6 + kubectl_bin exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.S1gMLUKQr7 ++ mktemp + local LAST_ERR=/tmp/tmp.StKPq8sGbz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S1gMLUKQr7 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("82a5bbb2-c03b-4571-a684-a833718d1a01") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.StKPq8sGbz + rm /tmp/tmp.S1gMLUKQr7 /tmp/tmp.StKPq8sGbz + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-15995 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-15995 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-15995 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-15995 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KOGZWrV3fH +++ mktemp ++ local LAST_ERR=/tmp/tmp.NpacwBexWc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KOGZWrV3fH ++ cat /tmp/tmp.NpacwBexWc ++ rm /tmp/tmp.KOGZWrV3fH /tmp/tmp.NpacwBexWc ++ return 0 + local client_container=psmdb-client-66f577db5f-lp7s6 + kubectl_bin exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.4WEbivoZmb ++ mktemp + local LAST_ERR=/tmp/tmp.9gaB4XODEr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-lp7s6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-15995.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4WEbivoZmb + cat /tmp/tmp.9gaB4XODEr + rm /tmp/tmp.4WEbivoZmb /tmp/tmp.9gaB4XODEr + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/compare/find.json /tmp/tmp.rLtYsl4wGo/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2025-05-21T10:34:41.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2025-05-21T10:34:45.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2025-05-21T10:34:44.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2025-05-21T10:34:46.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2025-05-21T10:34:45.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2025-05-21T10:34:44.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2025-05-21T10:34:43.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2025-05-21T10:34:45.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2025-05-21T10:34:45.000+0000 I listening for the commands + echo 'Sleeping for 360 seconds' Sleeping for 360 seconds + sleep 360 + backup_name_minio=backup-minio + run_backup backup-minio 2 physical + local name=backup-minio + local idx=2 + local type=physical + desc 'run backup backup-minio-2' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-2 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/pitr-physical/conf/backup-minio.yml + /usr/bin/sed -e 's/type:/type: physical/' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's/name:/name: backup-minio-2/' + local LAST_OUT=/tmp/tmp.sfFgalVvRe ++ mktemp + local LAST_ERR=/tmp/tmp.hJmi5uZWO5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sfFgalVvRe perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.hJmi5uZWO5 + rm /tmp/tmp.sfFgalVvRe /tmp/tmp.hJmi5uZWO5 + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state..........................................................2025-05-21T10:39:51.967Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "858244f6-f535-42ff-b004-d5ec403e2003"} 2025-05-21T10:40:02.333Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "d16d74ea-7adc-4b75-974e-9056e86b225b"} 2025-05-21T10:40:11.918Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "cd6d9585-f711-4f15-ad9c-0d20c9826981"} 2025-05-21T10:40:22.121Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "3aa6ec8b-51b3-40eb-97da-2aed67d14c93"} 2025-05-21T10:40:32.272Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "89e3694e-13c4-4f1e-90b0-9284b681b294"} 2025-05-21T10:40:41.980Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "66681551-ac78-4a83-b665-1937f51c86cc"} 2025-05-21T10:40:51.692Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "a975b399-ac59-4641-a450-247993867fdd"} 2025-05-21T10:41:01.890Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "3e62c2ca-2d24-40ec-8430-8252f41e3fd3"} 2025-05-21T10:41:11.785Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "0681adf8-ef5f-4eeb-a37d-71e4e57fb266"} 2025-05-21T10:41:21.590Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "d68e93ce-4947-401e-a8d1-72dc0759aacb"} 2025-05-21T10:41:30.338Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1"} 2025-05-21T10:41:30.623Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:30.623Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:30.731Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1"} 2025-05-21T10:41:30.892Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T10:41:30.892Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "backup": "backup-minio-2", "storage": "minio"} 2025-05-21T10:41:30.892Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "backup": "backup-minio-2", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T10:41:30Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T10:41:30.898Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1", "previous": "", "current": "requested"} 2025-05-21T10:41:30.924Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "40e3bd66-c2bd-47ac-aa39-728a00cbcfa1"} 2025-05-21T10:41:30.924Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3"} 2025-05-21T10:41:31.012Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:31.012Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:31.095Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3"} 2025-05-21T10:41:31.096Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3", "backup": "backup-minio-2", "storage": "minio"} 2025-05-21T10:41:31.096Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3", "backup": "backup-minio-2", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T10:41:31Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T10:41:31.102Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3", "previous": "", "current": "requested"} 2025-05-21T10:41:31.154Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "ed32879f-f905-4a8a-9a05-ea57522147c3"} 2025-05-21T10:41:31.154Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7"} 2025-05-21T10:41:31.227Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:31.228Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:31.330Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7"} 2025-05-21T10:41:31.331Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7", "pbmName": "2025-05-21T10:41:30Z", "backup": "backup-minio-2"} 2025-05-21T10:41:31.332Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d2cdd7-06a8-4e83-a3d0-0dd742be32a7"} 2025-05-21T10:41:31.332Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560"} 2025-05-21T10:41:31.461Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:31.461Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:31.557Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560"} 2025-05-21T10:41:31.559Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824091,"I":65},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:31.561Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "27dd8870-599f-4243-a23a-b967078bd560"} 2025-05-21T10:41:32.236Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "19f4da40-6daa-4ad2-8099-acc9c9b5c524"} 2025-05-21T10:41:35.925Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9"} 2025-05-21T10:41:36.014Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:36.014Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:36.109Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9"} 2025-05-21T10:41:36.110Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824091,"I":65},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:36.113Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "948517cc-b030-454b-8bb2-4910bd9bf6c9"} 2025-05-21T10:41:41.114Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2"} 2025-05-21T10:41:41.206Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:41.206Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:41.288Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2"} 2025-05-21T10:41:41.289Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:41.292Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "4b53ae24-61c5-472e-92ec-bcf27f72aad2"} 2025-05-21T10:41:41.639Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "0ff1265e-acd6-4d3a-a40a-ab43967ed372"} 2025-05-21T10:41:46.292Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1"} 2025-05-21T10:41:46.366Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:46.366Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:46.444Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1"} 2025-05-21T10:41:46.445Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:46.448Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "14bd96f0-4e3d-4ad0-83f0-de4be22e63b1"} 2025-05-21T10:41:50.862Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "9a2f0db7-e1eb-4c9e-8780-1b1b57daba5c"} 2025-05-21T10:41:51.448Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511"} 2025-05-21T10:41:51.529Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:51.529Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:51.614Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511"} 2025-05-21T10:41:51.615Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:51.616Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "681d58dc-32d3-4ac8-aafd-5a2e312b4511"} 2025-05-21T10:41:56.617Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c"} 2025-05-21T10:41:56.726Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:41:56.726Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:41:56.805Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c"} 2025-05-21T10:41:56.806Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:41:56.808Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "16855f99-4c59-4f8a-b715-9ab1dd72c33c"} 2025-05-21T10:42:01.810Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb"} 2025-05-21T10:42:01.890Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:01.890Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:01.978Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb"} 2025-05-21T10:42:01.979Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:01.981Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "645f275c-48f1-4e48-b996-baf6296d76eb"} 2025-05-21T10:42:04.293Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "bb587c8c-4599-44e8-bf92-06d0a95aee6f"} 2025-05-21T10:42:06.981Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7"} 2025-05-21T10:42:07.061Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:07.061Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:07.138Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7"} 2025-05-21T10:42:07.139Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:07.140Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "bdd26174-9da1-421f-86f4-07040222f4a7"} 2025-05-21T10:42:12.141Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe"} 2025-05-21T10:42:12.233Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:12.234Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:12.311Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe"} 2025-05-21T10:42:12.312Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:12.314Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "f7739b34-9806-4463-9ef7-5e8010e059fe"} 2025-05-21T10:42:13.672Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "aa026ff7-0b8f-4b30-9023-8bd591c389c4"} 2025-05-21T10:42:17.314Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f"} 2025-05-21T10:42:17.393Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:17.393Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:17.465Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f"} 2025-05-21T10:42:17.466Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:17.468Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "291dc527-626c-4e59-96a4-8e2aecc5385f"} 2025-05-21T10:42:22.469Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7"} 2025-05-21T10:42:22.554Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:22.554Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:22.638Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7"} 2025-05-21T10:42:22.639Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:22.641Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e95f6b21-fa0a-4e67-b70f-1e3e0922cda7"} 2025-05-21T10:42:23.314Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "c62529ce-2471-4439-9161-3a4b10701504"} 2025-05-21T10:42:27.642Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859"} 2025-05-21T10:42:27.735Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:27.735Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:27.820Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859"} 2025-05-21T10:42:27.821Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:27.823Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "50a6c4eb-80ab-4a75-b647-0846e766b859"} 2025-05-21T10:42:32.823Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4"} 2025-05-21T10:42:32.913Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:32.913Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:33.006Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4"} 2025-05-21T10:42:33.007Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:33.009Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d4122d1b-daac-4f75-9e91-2971ed8312b4"} 2025-05-21T10:42:33.801Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "39302254-4918-43a9-bc18-bdc3780f376c"} 2025-05-21T10:42:38.010Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8"} 2025-05-21T10:42:38.092Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:38.092Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:38.167Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8"} 2025-05-21T10:42:38.168Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:38.170Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "04be51d2-0e20-42da-80ba-7b6bafa974b8"} 2025-05-21T10:42:43.171Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac"} 2025-05-21T10:42:43.213Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "7f202451-8c06-4d20-b8d0-cb408a79c7e7"} 2025-05-21T10:42:43.259Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:43.259Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:43.343Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac"} 2025-05-21T10:42:43.343Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:43.349Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7d5ffd02-ea6f-47e2-bbf0-f2294b94eeac"} 2025-05-21T10:42:48.351Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde"} 2025-05-21T10:42:48.427Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:48.427Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:48.499Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde"} 2025-05-21T10:42:48.500Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:48.503Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7af28d4c-878e-4f69-887d-cb7f1e8bfbde"} 2025-05-21T10:42:52.838Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "26955733-8bd5-45c9-b74d-1ccd24af6499"} 2025-05-21T10:42:53.504Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce"} 2025-05-21T10:42:53.588Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:53.588Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:53.677Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce"} 2025-05-21T10:42:53.679Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:53.681Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "a69fac1a-33ce-4eb5-a5ec-9010d6285bce"} 2025-05-21T10:42:58.682Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58"} 2025-05-21T10:42:58.765Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:42:58.765Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:42:58.846Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58"} 2025-05-21T10:42:58.847Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:42:58.849Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "3fbe6d85-a693-4528-948e-0d26adb55c58"} 2025-05-21T10:43:02.996Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "52bd5b34-0e65-4c40-aff4-d735d29fd818"} 2025-05-21T10:43:03.850Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5"} 2025-05-21T10:43:03.933Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:03.933Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:04.026Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5"} 2025-05-21T10:43:04.027Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:04.030Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "b2f7e391-5c27-40c3-b7f9-bcb6924d6fb5"} 2025-05-21T10:43:09.030Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2"} 2025-05-21T10:43:09.120Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:09.121Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:09.209Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2"} 2025-05-21T10:43:09.211Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:09.213Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "87234760-6dd0-493b-93c8-29a62dfe40f2"} 2025-05-21T10:43:12.664Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "e9f7d4cf-200e-47b9-9239-7698d4c89444"} 2025-05-21T10:43:14.214Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e"} 2025-05-21T10:43:14.293Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:14.293Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:14.373Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e"} 2025-05-21T10:43:14.374Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:14.375Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "82d8a4c3-cfff-49d8-ba0c-ded9124f3c8e"} 2025-05-21T10:43:19.377Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9"} 2025-05-21T10:43:19.449Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:19.449Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:19.527Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9"} 2025-05-21T10:43:19.528Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:19.530Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e007e684-2c75-41bb-913d-ebeece5395a9"} 2025-05-21T10:43:22.212Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "c377529b-6f20-4716-a2c9-4c1bc6555137"} 2025-05-21T10:43:24.531Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e"} 2025-05-21T10:43:24.609Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:24.609Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:24.722Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e"} 2025-05-21T10:43:24.723Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:24.725Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "7167eb24-35c3-44e5-90c7-28e5f2fb4e2e"} 2025-05-21T10:43:29.726Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6"} 2025-05-21T10:43:29.805Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:29.806Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:29.880Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6"} 2025-05-21T10:43:29.881Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:29.882Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "e2a77826-8c4b-4607-95c6-3f35f2e679f6"} 2025-05-21T10:43:32.360Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "some-name", "reconcileID": "4fcca6bb-09b9-4e33-aea6-81722772db93"} 2025-05-21T10:43:34.883Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9"} 2025-05-21T10:43:34.962Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9", "cluster": "some-name", "namespace": "pitr-physical-15995"} 2025-05-21T10:43:34.962Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9", "currentJob": {"Name":"backup-minio-2","Type":0}} 2025-05-21T10:43:35.039Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9"} 2025-05-21T10:43:35.040Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9", "backup": "backup-minio-2", "pbmName": "2025-05-21T10:41:31Z", "meta": {"type":"physical","opid":"682daddbb2a12b7a6b5a5000","name":"2025-05-21T10:41:31Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747824091,"last_transition_ts":1747824091,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747824096,"I":36},"status":"starting","conditions":[{"timestamp":1747824091,"status":"starting"}],"n":[{"rs":"rs1","n":["some-name-rs1-0.some-name-rs1.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.pitr-physical-15995.svc.cluster.local:27017"],"ack":""},{"rs":"rs2","n":["some-name-rs2-0.some-name-rs2.pitr-physical-15995.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T10:43:35.042Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9", "previous": "requested", "current": "error"} 2025-05-21T10:43:35.060Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T10:43:35.090Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "274b3c5b-d8ea-44f4-a326-d71b3de3d9f9"} 2025-05-21T10:43:35.090Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d8e25636-80e0-4e97-9748-fa28011777e4"} 2025-05-21T10:43:35.090Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-2","namespace":"pitr-physical-15995"}, "namespace": "pitr-physical-15995", "name": "backup-minio-2", "reconcileID": "d8e25636-80e0-4e97-9748-fa28011777e4"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-minio-2 some-name minio s3://operator-testing/2025-05-21T10:41:31Z physical error 2m10s Backup object psmdb-backup/backup-minio-2 is in error state. something went wrong with operator or kubernetes cluster