Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/logs/demand-backup-physical-sharded.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + create_infra demand-backup-physical-sharded-18556 + local ns=demand-backup-physical-sharded-18556 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.TwsYKYyrrG ++ mktemp + local LAST_ERR=/tmp/tmp.aEzcBRSr1K + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TwsYKYyrrG customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.aEzcBRSr1K + rm /tmp/tmp.TwsYKYyrrG /tmp/tmp.aEzcBRSr1K + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-76 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-azure-blob-sharded" not found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-76 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.erEfUqDk10 ++ mktemp + local LAST_ERR=/tmp/tmp.ocksNWOiqd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.erEfUqDk10 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.ocksNWOiqd + rm /tmp/tmp.erEfUqDk10 /tmp/tmp.ocksNWOiqd + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.E7kZzpgjSD ++ mktemp + local LAST_ERR=/tmp/tmp.927ZF39PYC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E7kZzpgjSD + cat /tmp/tmp.927ZF39PYC + rm /tmp/tmp.E7kZzpgjSD /tmp/tmp.927ZF39PYC + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.6O4o0WTJBV ++ mktemp + local LAST_ERR=/tmp/tmp.LvLfZBn41i + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6O4o0WTJBV + cat /tmp/tmp.LvLfZBn41i + rm /tmp/tmp.6O4o0WTJBV /tmp/tmp.LvLfZBn41i + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.wFC8mxHbBZ ++ mktemp + local LAST_ERR=/tmp/tmp.TZQ3lE7VjW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wFC8mxHbBZ clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.TZQ3lE7VjW + rm /tmp/tmp.wFC8mxHbBZ /tmp/tmp.TZQ3lE7VjW + return 0 + check_crd_for_deletion PR-1926-2614d852 + local git_tag=PR-1926-2614d852 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1926-2614d852/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/bin/sed s/---//g + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C4FO06rhZ1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.eORoff3avp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.C4FO06rhZ1 ++ cat /tmp/tmp.eORoff3avp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.C4FO06rhZ1 ++ cat /tmp/tmp.eORoff3avp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.C4FO06rhZ1 ++ cat /tmp/tmp.eORoff3avp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.C4FO06rhZ1 ++ cat /tmp/tmp.eORoff3avp Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.C4FO06rhZ1 /tmp/tmp.eORoff3avp ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + awk '{print$1}' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.jTN7PhlfOK ++ mktemp + local LAST_OUT=/tmp/tmp.YgE4Yo56fO ++ mktemp + local LAST_ERR=/tmp/tmp.6gybGT9hhb + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.XmbLA6YuW9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jTN7PhlfOK + cat /tmp/tmp.6gybGT9hhb + rm /tmp/tmp.jTN7PhlfOK /tmp/tmp.6gybGT9hhb + return 0 namespace "demand-backup-physical-sharded-76" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YgE4Yo56fO namespace "psmdb-operator" deleted + cat /tmp/tmp.XmbLA6YuW9 + rm /tmp/tmp.YgE4Yo56fO /tmp/tmp.XmbLA6YuW9 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.V71t0p07aV ++ mktemp + local LAST_ERR=/tmp/tmp.dJAb8gmOsw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V71t0p07aV + cat /tmp/tmp.dJAb8gmOsw + rm /tmp/tmp.V71t0p07aV /tmp/tmp.dJAb8gmOsw + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mFJjp2NT07 ++ mktemp + local LAST_ERR=/tmp/tmp.RIRjJz0stn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mFJjp2NT07 namespace/psmdb-operator created + cat /tmp/tmp.RIRjJz0stn + rm /tmp/tmp.mFJjp2NT07 /tmp/tmp.RIRjJz0stn + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.arIjliEpGt +++ mktemp ++ local LAST_ERR=/tmp/tmp.CxyRJUqHQl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.arIjliEpGt ++ cat /tmp/tmp.CxyRJUqHQl ++ rm /tmp/tmp.arIjliEpGt /tmp/tmp.CxyRJUqHQl ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.kYif3yU4uh ++ mktemp + local LAST_ERR=/tmp/tmp.lV8VjJxyZK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kYif3yU4uh Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1" modified. + cat /tmp/tmp.lV8VjJxyZK + rm /tmp/tmp.kYif3yU4uh /tmp/tmp.lV8VjJxyZK + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.4403o43Gd4 ++ mktemp + local LAST_ERR=/tmp/tmp.xPsb0lficR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4403o43Gd4 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.xPsb0lficR + rm /tmp/tmp.4403o43Gd4 /tmp/tmp.xPsb0lficR + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rvRer7A1Yb ++ mktemp + local LAST_ERR=/tmp/tmp.UmGbS1QOy4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rvRer7A1Yb clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.UmGbS1QOy4 + rm /tmp/tmp.rvRer7A1Yb /tmp/tmp.UmGbS1QOy4 + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1926-2614d852") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.8RPlr9PB0F ++ mktemp + local LAST_ERR=/tmp/tmp.a5IfIHdIpb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8RPlr9PB0F deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.a5IfIHdIpb + rm /tmp/tmp.8RPlr9PB0F /tmp/tmp.a5IfIHdIpb + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.8FOh3tzG6H +++ mktemp ++ local LAST_ERR=/tmp/tmp.9thpaFtqXi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8FOh3tzG6H ++ cat /tmp/tmp.9thpaFtqXi ++ rm /tmp/tmp.8FOh3tzG6H /tmp/tmp.9thpaFtqXi ++ return 0 + wait_pod percona-server-mongodb-operator-87cfc696-s6zpx + local pod=percona-server-mongodb-operator-87cfc696-s6zpx + set +o xtrace waiting for pod/percona-server-mongodb-operator-87cfc696-s6zpx to be ready.OK + echo 'Print operator info from log' Print operator info from log ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator + grep 'Manager starting up' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aKxBfjQKOd +++ mktemp ++ local LAST_ERR=/tmp/tmp.nAJInKKhqc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aKxBfjQKOd ++ cat /tmp/tmp.nAJInKKhqc ++ rm /tmp/tmp.aKxBfjQKOd /tmp/tmp.nAJInKKhqc ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-87cfc696-s6zpx ++ mktemp + local LAST_OUT=/tmp/tmp.lJ701Qt3Bg ++ mktemp + local LAST_ERR=/tmp/tmp.SU4DVn6EkT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-87cfc696-s6zpx + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lJ701Qt3Bg + cat /tmp/tmp.SU4DVn6EkT + rm /tmp/tmp.lJ701Qt3Bg /tmp/tmp.SU4DVn6EkT + return 0 2025-05-21T22:41:49.718Z INFO setup Manager starting up {"gitCommit": "2614d8526fb6cb612447e95aa985aca9a64a3da0", "gitBranch": "PR-1926-2614d852", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-sharded-18556 + local namespace=demand-backup-physical-sharded-18556 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-physical-sharded-18556' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-sharded-18556 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-sharded-18556 --ignore-not-found + awk '{print$1}' ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.kSqucm5nbV + local LAST_OUT=/tmp/tmp.f5YckqsIkb ++ mktemp + local LAST_ERR=/tmp/tmp.QbcrwEqcSl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-physical-sharded-18556 --ignore-not-found + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.HfNhMCQzIK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kSqucm5nbV + cat /tmp/tmp.HfNhMCQzIK + rm /tmp/tmp.kSqucm5nbV /tmp/tmp.HfNhMCQzIK + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f5YckqsIkb + cat /tmp/tmp.QbcrwEqcSl + rm /tmp/tmp.f5YckqsIkb /tmp/tmp.QbcrwEqcSl + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-18556 ++ mktemp + local LAST_OUT=/tmp/tmp.rj87UMBJXA ++ mktemp + local LAST_ERR=/tmp/tmp.hv4OQ7GCOF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace demand-backup-physical-sharded-18556 namespace "gke-managed-cim" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rj87UMBJXA + cat /tmp/tmp.hv4OQ7GCOF + rm /tmp/tmp.rj87UMBJXA /tmp/tmp.hv4OQ7GCOF + return 0 + desc 'create namespace demand-backup-physical-sharded-18556' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-sharded-18556 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-sharded-18556 ++ mktemp + local LAST_OUT=/tmp/tmp.iIvXxMfvfI ++ mktemp + local LAST_ERR=/tmp/tmp.uZEGKH8J0Z + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace demand-backup-physical-sharded-18556 namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iIvXxMfvfI namespace/demand-backup-physical-sharded-18556 created + cat /tmp/tmp.uZEGKH8J0Z + rm /tmp/tmp.iIvXxMfvfI /tmp/tmp.uZEGKH8J0Z + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.uFzGGJAVEN +++ mktemp ++ local LAST_ERR=/tmp/tmp.lPLMyombKq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uFzGGJAVEN ++ cat /tmp/tmp.lPLMyombKq ++ rm /tmp/tmp.uFzGGJAVEN /tmp/tmp.lPLMyombKq ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1 --namespace=demand-backup-physical-sharded-18556 ++ mktemp + local LAST_OUT=/tmp/tmp.kQeZixabQM ++ mktemp + local LAST_ERR=/tmp/tmp.Cnf6EMPn7A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1 --namespace=demand-backup-physical-sharded-18556 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kQeZixabQM Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster1" modified. + cat /tmp/tmp.Cnf6EMPn7A + rm /tmp/tmp.kQeZixabQM /tmp/tmp.Cnf6EMPn7A + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Wed May 21 22:42:15 2025 NAMESPACE: demand-backup-physical-sharded-18556 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-physical-sharded-18556.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-sharded-18556 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-sharded-18556 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-sharded-18556 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-sharded-18556 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PCjbJXTka6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.7KoDTsPM7j ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PCjbJXTka6 ++ cat /tmp/tmp.7KoDTsPM7j ++ rm /tmp/tmp.PCjbJXTka6 /tmp/tmp.7KoDTsPM7j ++ return 0 + MINIO_POD=minio-service-86dfccd949-rjm8d + wait_pod minio-service-86dfccd949-rjm8d + local pod=minio-service-86dfccd949-rjm8d + set +o xtrace waiting for pod/minio-service-86dfccd949-rjm8d to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-18556.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.XsnzFviSxT ++ mktemp + local LAST_ERR=/tmp/tmp.xltwCtxdbA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-18556.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XsnzFviSxT service/minio-service created + cat /tmp/tmp.xltwCtxdbA + rm /tmp/tmp.XsnzFviSxT /tmp/tmp.xltwCtxdbA + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.i4NmqpNmC4 ++ mktemp + local LAST_ERR=/tmp/tmp.ust0mBLjaZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.i4NmqpNmC4 make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.ust0mBLjaZ + rm /tmp/tmp.i4NmqpNmC4 /tmp/tmp.ust0mBLjaZ + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.iU1r9EAuLu ++ mktemp + local LAST_ERR=/tmp/tmp.dXKq1jN5ci + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iU1r9EAuLu secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.dXKq1jN5ci + rm /tmp/tmp.iU1r9EAuLu /tmp/tmp.dXKq1jN5ci + return 0 + desc 'Testing on sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on sharded cluster ----------------------------------------------------------------------------------- + echo 'Creating PSMDB cluster' Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MFFC0sBOPs ++ mktemp + local LAST_ERR=/tmp/tmp.dAcsyS3TKW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MFFC0sBOPs secret/some-users created + cat /tmp/tmp.dAcsyS3TKW + rm /tmp/tmp.MFFC0sBOPs /tmp/tmp.dAcsyS3TKW + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1926-2614d852"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + local LAST_OUT=/tmp/tmp.KmvqAtyrAi + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_ERR=/tmp/tmp.GXp5BuBtKk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KmvqAtyrAi perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.GXp5BuBtKk + rm /tmp/tmp.KmvqAtyrAi /tmp/tmp.GXp5BuBtKk + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4ThAqzQJDw ++ mktemp + local LAST_ERR=/tmp/tmp.1dox2zEufE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4ThAqzQJDw deployment.apps/psmdb-client created + cat /tmp/tmp.1dox2zEufE + rm /tmp/tmp.4ThAqzQJDw /tmp/tmp.1dox2zEufE + return 0 + echo 'check if all pods started' check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.....OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lBBWxF4Cmd +++ mktemp ++ local LAST_ERR=/tmp/tmp.kqqIJelC7U ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lBBWxF4Cmd ++ cat /tmp/tmp.kqqIJelC7U ++ rm /tmp/tmp.lBBWxF4Cmd /tmp/tmp.kqqIJelC7U ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l2tz5r8PLi +++ mktemp ++ local LAST_ERR=/tmp/tmp.kUewo6XbDF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.l2tz5r8PLi ++ cat /tmp/tmp.kUewo6XbDF ++ rm /tmp/tmp.l2tz5r8PLi /tmp/tmp.kUewo6XbDF ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dm0OJLe2jj +++ mktemp ++ local LAST_ERR=/tmp/tmp.343eTVc3ik ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Dm0OJLe2jj ++ cat /tmp/tmp.343eTVc3ik ++ rm /tmp/tmp.Dm0OJLe2jj /tmp/tmp.343eTVc3ik ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.alS7wq1YXY +++ mktemp ++ local LAST_ERR=/tmp/tmp.sDKZ6Nnm6H ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.alS7wq1YXY ++ cat /tmp/tmp.sDKZ6Nnm6H ++ rm /tmp/tmp.alS7wq1YXY /tmp/tmp.sDKZ6Nnm6H ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qJqNOCgvDm +++ mktemp ++ local LAST_ERR=/tmp/tmp.GAFC0TSsQV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qJqNOCgvDm ++ cat /tmp/tmp.GAFC0TSsQV ++ rm /tmp/tmp.qJqNOCgvDm /tmp/tmp.GAFC0TSsQV ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ac635VOMbM +++ mktemp ++ local LAST_ERR=/tmp/tmp.4rksjyskO5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ac635VOMbM ++ cat /tmp/tmp.4rksjyskO5 ++ rm /tmp/tmp.ac635VOMbM /tmp/tmp.4rksjyskO5 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uYLbi6w5GR +++ mktemp ++ local LAST_ERR=/tmp/tmp.ettx9rs7nG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uYLbi6w5GR ++ cat /tmp/tmp.ettx9rs7nG ++ rm /tmp/tmp.uYLbi6w5GR /tmp/tmp.ettx9rs7nG ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo ++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CrEFjCRjsm +++ mktemp ++ local LAST_ERR=/tmp/tmp.xATghLWO8S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get svc some-name-mongos '-o=jsonpath={.status}' ++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CrEFjCRjsm ++ cat /tmp/tmp.xATghLWO8S ++ rm /tmp/tmp.CrEFjCRjsm /tmp/tmp.xATghLWO8S ++ return 0 + lbEndpoint=34.46.91.152 + '[' -z 34.46.91.152 ']' + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-18556 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-18556 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OQ5Y1MKF9a +++ mktemp ++ local LAST_ERR=/tmp/tmp.ftnfcRlLtF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OQ5Y1MKF9a ++ cat /tmp/tmp.ftnfcRlLtF ++ rm /tmp/tmp.OQ5Y1MKF9a /tmp/tmp.ftnfcRlLtF ++ return 0 + local client_container=psmdb-client-b9788d8bc-x2kjt + kubectl_bin exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.8mHiK9BuRG ++ mktemp + local LAST_ERR=/tmp/tmp.qt947xoyMl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8mHiK9BuRG Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("61c7313b-4988-4fa9-8c25-05b881ffae48") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.qt947xoyMl + rm /tmp/tmp.8mHiK9BuRG /tmp/tmp.qt947xoyMl + return 0 + sleep 1 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uJjWyEZzfg +++ mktemp ++ local LAST_ERR=/tmp/tmp.kyyQKtQPiz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uJjWyEZzfg ++ cat /tmp/tmp.kyyQKtQPiz ++ rm /tmp/tmp.uJjWyEZzfg /tmp/tmp.kyyQKtQPiz ++ return 0 + local client_container=psmdb-client-b9788d8bc-x2kjt + kubectl_bin exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.HGzMRNBIyq ++ mktemp + local LAST_ERR=/tmp/tmp.9S67qBOdZz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HGzMRNBIyq Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("522fb809-2686-4c94-b3e7-e4751e4cbe03") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.9S67qBOdZz + rm /tmp/tmp.HGzMRNBIyq /tmp/tmp.9S67qBOdZz + return 0 + sleep 5 + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 -sharded + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 + local postfix=-sharded + local suffix= + local database=myApp + local collection=test + local port=27017 + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.P6hM6Bllfx +++ mktemp ++ local LAST_ERR=/tmp/tmp.tA2BK2niRI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.P6hM6Bllfx ++ cat /tmp/tmp.tA2BK2niRI ++ rm /tmp/tmp.P6hM6Bllfx /tmp/tmp.tA2BK2niRI ++ return 0 + local client_container=psmdb-client-b9788d8bc-x2kjt + kubectl_bin exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.gMneDLpvko ++ mktemp + local LAST_ERR=/tmp/tmp.TDjWYNRdhM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-x2kjt -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-18556.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gMneDLpvko + cat /tmp/tmp.TDjWYNRdhM + rm /tmp/tmp.gMneDLpvko /tmp/tmp.TDjWYNRdhM + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.Fns89lK40L/find-sharded + echo 'waiting 60 seconds for stable timestamp in wiredtiger' waiting 60 seconds for stable timestamp in wiredtiger + sleep 80 + echo 'running backups' running backups + backup_name_minio=backup-minio-sharded + run_backup minio backup-minio-sharded + local storage=minio + local backup_name=backup-minio-sharded + /usr/bin/sed -e 's/name:/name: backup-minio-sharded/' + /usr/bin/sed -e 's/storageName:/storageName: minio/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.bNfTS6FgAC ++ mktemp + local LAST_ERR=/tmp/tmp.kqVnzWPITJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bNfTS6FgAC perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created + cat /tmp/tmp.kqVnzWPITJ + rm /tmp/tmp.bNfTS6FgAC /tmp/tmp.kqVnzWPITJ + return 0 + '[' -z '' ']' + backup_name_aws=backup-aws-s3-sharded + backup_name_gcp=backup-gcp-cs-sharded + backup_name_azure=backup-azure-blob-sharded + run_backup aws-s3 backup-aws-s3-sharded + local storage=aws-s3 + local backup_name=backup-aws-s3-sharded + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/backup.yml + /usr/bin/sed -e 's/storageName:/storageName: aws-s3/' + /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.YgLG8GYUsJ ++ mktemp + local LAST_ERR=/tmp/tmp.7nd4gj3wyS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YgLG8GYUsJ perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created + cat /tmp/tmp.7nd4gj3wyS + rm /tmp/tmp.YgLG8GYUsJ /tmp/tmp.7nd4gj3wyS + return 0 + run_backup gcp-cs backup-gcp-cs-sharded + local storage=gcp-cs + local backup_name=backup-gcp-cs-sharded + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/backup.yml + /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/' + /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tDqV6C56bi ++ mktemp + local LAST_ERR=/tmp/tmp.vdUKVBHDlv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tDqV6C56bi perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created + cat /tmp/tmp.vdUKVBHDlv + rm /tmp/tmp.tDqV6C56bi /tmp/tmp.vdUKVBHDlv + return 0 + run_backup azure-blob backup-azure-blob-sharded + local storage=azure-blob + local backup_name=backup-azure-blob-sharded + /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/' + /usr/bin/sed -e 's/storageName:/storageName: azure-blob/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-physical-sharded/conf/backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.14Lo5hQSsG ++ mktemp + local LAST_ERR=/tmp/tmp.iwrPn6Nmez + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.14Lo5hQSsG perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created + cat /tmp/tmp.iwrPn6Nmez + rm /tmp/tmp.14Lo5hQSsG /tmp/tmp.iwrPn6Nmez + return 0 + wait_backup backup-aws-s3-sharded + local backup_name=backup-aws-s3-sharded + local target_state=ready + set +o xtrace waiting for backup-aws-s3-sharded to reach ready state........................................................................................................................... + check_backup_in_storage backup-aws-s3-sharded s3 rs0 + local backup=backup-aws-s3-sharded + local storage_type=s3 + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=s3.amazonaws.com ++ get_backup_dest backup-aws-s3-sharded ++ local backup_name=backup-aws-s3-sharded ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed -e 's/.json$//' ++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8HElxKkh6f +++ mktemp ++ local LAST_ERR=/tmp/tmp.vSGgj83eXn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8HElxKkh6f ++ cat /tmp/tmp.vSGgj83eXn ++ rm /tmp/tmp.8HElxKkh6f /tmp/tmp.vSGgj83eXn ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z + local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/rs0/filelist.pbm + log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/rs0/filelist.pbm exists' + set +o xtrace [2025-05-21T22:52:17+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/rs0/filelist.pbm exists + curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 12822 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK x-amz-id-2: +cGai9f6BDO/hvsL9UQrANOByPF7/ma7Ulffv0Iy4xd0gF8LU3qGZLMUHKz2So9iCoRC0mYK1ws= x-amz-request-id: JTMNRBYWNV7T5K72 Date: Wed, 21 May 2025 22:52:19 GMT Last-Modified: Wed, 21 May 2025 22:52:06 GMT x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup" ETag: "884fe1ab75a5f5120a0749841ad871f9" x-amz-server-side-encryption: AES256 Accept-Ranges: bytes Content-Type: binary/octet-stream Content-Length: 12822 Server: AmazonS3 + check_backup_in_storage backup-aws-s3-sharded s3 cfg + local backup=backup-aws-s3-sharded + local storage_type=s3 + local replset=cfg + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=s3.amazonaws.com ++ get_backup_dest backup-aws-s3-sharded ++ local backup_name=backup-aws-s3-sharded ++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OW9QBOVKBo +++ mktemp ++ local LAST_ERR=/tmp/tmp.38mSUKYTaA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|azure://||' ++ sed 's|s3://||' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OW9QBOVKBo ++ cat /tmp/tmp.38mSUKYTaA ++ rm /tmp/tmp.OW9QBOVKBo /tmp/tmp.38mSUKYTaA ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z + local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/cfg/filelist.pbm + log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/cfg/filelist.pbm exists' + set +o xtrace [2025-05-21T22:52:19+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/cfg/filelist.pbm exists + curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z/cfg/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 19099 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK x-amz-id-2: 1S7SOZfaZfg60TNB/mFnh92eMwlzSUzeODuacLbzpf+UqA4uykw2nlG4asKf4azDrniX2+B3pwI= x-amz-request-id: HGF92DJWW8J1G53Z Date: Wed, 21 May 2025 22:52:20 GMT Last-Modified: Wed, 21 May 2025 22:52:10 GMT x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup" ETag: "dbaff333b7f33e7c102bd5ff0bdffa38" x-amz-server-side-encryption: AES256 Accept-Ranges: bytes Content-Type: binary/octet-stream Content-Length: 19099 Server: AmazonS3 + wait_backup backup-gcp-cs-sharded + local backup_name=backup-gcp-cs-sharded + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sharded to reach ready state.2025-05-21T22:51:12.335Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "6aa48b30-0944-47a9-8f97-d4b3054b6000", "previous": "requested", "current": "running"} 2025-05-21T22:51:12.391Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "6aa48b30-0944-47a9-8f97-d4b3054b6000"} 2025-05-21T22:51:12.392Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390"} 2025-05-21T22:51:12.484Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:12.484Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T22:51:12.564Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390"} 2025-05-21T22:51:12.566Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T22:51:07Z", "meta": {"type":"physical","opid":"682e58db40f10cb361f560f6","name":"2025-05-21T22:51:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867867,"status":"running","last_transition_ts":1747867867,"first_write_ts":{"T":1747867861,"I":1},"last_write_ts":{"T":1747867861,"I":1},"node":"some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867867,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867867,"status":"running","iscs":true,"last_transition_ts":1747867867,"first_write_ts":{"T":1747867869,"I":1},"last_write_ts":{"T":1747867869,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867867,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867867,"last_transition_ts":1747867870,"first_write_ts":{"T":1747867869,"I":1},"last_write_ts":{"T":1747867869,"I":2},"hb":{"T":1747867867,"I":33},"status":"running","conditions":[{"timestamp":1747867867,"status":"starting"},{"timestamp":1747867870,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:12.569Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "c847baa5-94b7-4403-a2bf-6c172c411390"} 2025-05-21T22:51:13.985Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85"} 2025-05-21T22:51:14.078Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:14.078Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:14.160Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85"} 2025-05-21T22:51:14.160Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85", "holder": "backup-minio-sharded-06dc36f7-d3b0-4421-9b9c-7b20e8d260dc"} 2025-05-21T22:51:14.161Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f471f826-7b17-4cb3-b100-e5940610fd85"} 2025-05-21T22:51:14.492Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8"} 2025-05-21T22:51:14.708Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:14.708Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:14.826Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8"} 2025-05-21T22:51:14.826Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8", "holder": "backup-minio-sharded-06dc36f7-d3b0-4421-9b9c-7b20e8d260dc"} 2025-05-21T22:51:14.828Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2bf13013-c008-430d-8c21-fc0f9903f0e8"} 2025-05-21T22:51:17.392Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1"} 2025-05-21T22:51:17.458Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:17.458Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T22:51:17.522Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1"} 2025-05-21T22:51:17.523Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T22:51:07Z", "meta": {"type":"physical","opid":"682e58db40f10cb361f560f6","name":"2025-05-21T22:51:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867867,"status":"done","last_transition_ts":1747867874,"first_write_ts":{"T":1747867861,"I":1},"last_write_ts":{"T":1747867861,"I":1},"node":"some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867867,"status":"running"},{"timestamp":1747867874,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867867,"status":"done","iscs":true,"last_transition_ts":1747867874,"first_write_ts":{"T":1747867869,"I":1},"last_write_ts":{"T":1747867869,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867867,"status":"running"},{"timestamp":1747867874,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":2845970,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867867,"last_transition_ts":1747867875,"first_write_ts":{"T":1747867869,"I":1},"last_write_ts":{"T":1747867869,"I":2},"hb":{"T":1747867872,"I":55},"status":"done","conditions":[{"timestamp":1747867867,"status":"starting"},{"timestamp":1747867870,"status":"running"},{"timestamp":1747867875,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:17.525Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1", "previous": "running", "current": "ready"} 2025-05-21T22:51:17.553Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T22:51:17.579Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "4bbbafa0-f091-4aa8-ba06-39c577f070c1"} 2025-05-21T22:51:17.579Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "6bac6d05-b8bf-4168-8af1-d616d6a22a71"} 2025-05-21T22:51:17.579Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "6bac6d05-b8bf-4168-8af1-d616d6a22a71"} 2025-05-21T22:51:19.162Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48"} 2025-05-21T22:51:19.237Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:19.237Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:19.311Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48"} 2025-05-21T22:51:19.365Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T22:51:19.365Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "backup": "backup-azure-blob-sharded", "storage": "azure-blob"} 2025-05-21T22:51:19.365Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "backup": "backup-azure-blob-sharded", "storage": "azure-blob", "backupCmd": "backup [name: 2025-05-21T22:51:19Z, compression: gzip (level: default)] ", "profile": "azure-blob"} 2025-05-21T22:51:19.371Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48", "previous": "waiting", "current": "requested"} 2025-05-21T22:51:19.430Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "af32f6ad-e6ed-4c65-be97-176e0e787a48"} 2025-05-21T22:51:19.430Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff"} 2025-05-21T22:51:19.507Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:19.508Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:19.591Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff"} 2025-05-21T22:51:19.591Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff", "backup": "backup-azure-blob-sharded", "storage": "azure-blob"} 2025-05-21T22:51:19.591Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff", "backup": "backup-azure-blob-sharded", "storage": "azure-blob", "backupCmd": "backup [name: 2025-05-21T22:51:19Z, compression: gzip (level: default)] ", "profile": "azure-blob"} 2025-05-21T22:51:19.597Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff", "previous": "waiting", "current": "requested"} 2025-05-21T22:51:19.614Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "fdd107ef-9331-4e32-8977-83f3ba5fdaff"} 2025-05-21T22:51:19.614Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12"} 2025-05-21T22:51:19.691Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:19.691Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:19.768Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12"} 2025-05-21T22:51:19.768Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12", "pbmName": "2025-05-21T22:51:19Z", "backup": "backup-azure-blob-sharded"} 2025-05-21T22:51:19.769Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "4dc7e928-b2d3-4ba4-9d36-ede86750eb12"} 2025-05-21T22:51:19.828Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd"} 2025-05-21T22:51:19.907Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:19.907Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:19.980Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd"} 2025-05-21T22:51:19.980Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd", "holder": "backup-azure-blob-sharded-3aea2a05-0bea-4a01-a158-802599de81dd"} 2025-05-21T22:51:19.981Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4c45a810-d322-4674-b281-587ab22f1bbd"} 2025-05-21T22:51:22.580Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "9aad050a-aae8-4d08-96fa-0de2e9358990"} 2025-05-21T22:51:22.580Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-minio-sharded", "reconcileID": "9aad050a-aae8-4d08-96fa-0de2e9358990"} 2025-05-21T22:51:24.431Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386"} 2025-05-21T22:51:24.497Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:24.497Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:24.563Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386"} 2025-05-21T22:51:24.564Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"running","last_transition_ts":1747867880,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"running","iscs":true,"last_transition_ts":1747867880,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867883,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867880,"I":31},"status":"running","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:24.566Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386", "previous": "requested", "current": "running"} 2025-05-21T22:51:24.638Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "f67c40da-e3e1-40f0-bd17-47da788ee386"} 2025-05-21T22:51:24.660Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040"} 2025-05-21T22:51:24.726Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:24.726Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:24.798Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040"} 2025-05-21T22:51:24.799Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"running","last_transition_ts":1747867880,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"running","iscs":true,"last_transition_ts":1747867880,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867883,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867880,"I":31},"status":"running","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:24.801Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "e82bac4a-adc3-48fc-85a5-7e9a82658040"} 2025-05-21T22:51:24.982Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6"} 2025-05-21T22:51:25.055Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:25.055Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:25.129Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6"} 2025-05-21T22:51:25.130Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6", "holder": "backup-azure-blob-sharded-3aea2a05-0bea-4a01-a158-802599de81dd"} 2025-05-21T22:51:25.131Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7badeb13-b47e-412f-86ac-d0a9be9a44b6"} 2025-05-21T22:51:29.639Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941"} 2025-05-21T22:51:29.703Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:29.703Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:29.771Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941"} 2025-05-21T22:51:29.772Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"running","last_transition_ts":1747867880,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"running","iscs":true,"last_transition_ts":1747867880,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867883,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867885,"I":6},"status":"running","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:29.774Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "60bb1f4b-9918-42dc-9947-7e5e066d9941"} 2025-05-21T22:51:30.132Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea"} 2025-05-21T22:51:30.199Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:30.199Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:30.266Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea"} 2025-05-21T22:51:30.266Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea", "holder": "backup-azure-blob-sharded-3aea2a05-0bea-4a01-a158-802599de81dd"} 2025-05-21T22:51:30.267Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6adc7b89-b80f-4fb2-87d0-aefa3ad7daea"} 2025-05-21T22:51:34.775Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad"} 2025-05-21T22:51:34.845Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:34.845Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:34.919Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad"} 2025-05-21T22:51:34.920Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"running","last_transition_ts":1747867880,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"running","iscs":true,"last_transition_ts":1747867880,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867883,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867890,"I":16},"status":"running","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:34.923Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "da226d4d-3418-4bea-8e10-79c18bb601ad"} 2025-05-21T22:51:35.268Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b"} 2025-05-21T22:51:35.341Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:35.341Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:35.414Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b"} 2025-05-21T22:51:35.414Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b", "holder": "backup-azure-blob-sharded-3aea2a05-0bea-4a01-a158-802599de81dd"} 2025-05-21T22:51:35.423Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6eb8ee25-dff4-43f2-beff-e19ccba8288b"} 2025-05-21T22:51:39.924Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb"} 2025-05-21T22:51:39.994Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:39.994Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:40.072Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb"} 2025-05-21T22:51:40.073Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"done","last_transition_ts":1747867899,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"},{"timestamp":1747867899,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"done","iscs":true,"last_transition_ts":1747867895,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"},{"timestamp":1747867895,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":3621928,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867883,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867895,"I":23},"status":"running","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:40.075Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27c6316b-9bf8-4c1c-9e1e-737879fcfeeb"} 2025-05-21T22:51:40.423Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f"} 2025-05-21T22:51:40.497Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:40.497Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:40.563Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f"} 2025-05-21T22:51:40.563Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f", "holder": "backup-azure-blob-sharded-3aea2a05-0bea-4a01-a158-802599de81dd"} 2025-05-21T22:51:40.564Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4ff99652-0dc0-4a90-9997-eb875020042f"} 2025-05-21T22:51:45.076Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b"} 2025-05-21T22:51:45.139Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:45.139Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T22:51:45.204Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b"} 2025-05-21T22:51:45.204Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T22:51:19Z", "meta": {"type":"physical","opid":"682e58e740f10cb361f56129","name":"2025-05-21T22:51:19Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867880,"status":"done","last_transition_ts":1747867899,"first_write_ts":{"T":1747867881,"I":2},"last_write_ts":{"T":1747867881,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"},{"timestamp":1747867899,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867880,"status":"done","iscs":true,"last_transition_ts":1747867895,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"node":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867880,"status":"running"},{"timestamp":1747867895,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded"}},"size":3621928,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867880,"last_transition_ts":1747867900,"first_write_ts":{"T":1747867882,"I":1},"last_write_ts":{"T":1747867882,"I":2},"hb":{"T":1747867895,"I":23},"status":"done","conditions":[{"timestamp":1747867880,"status":"starting"},{"timestamp":1747867883,"status":"running"},{"timestamp":1747867900,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:45.206Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b", "previous": "running", "current": "ready"} 2025-05-21T22:51:45.238Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T22:51:45.266Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "27307558-dd56-4221-a17b-13f11a07695b"} 2025-05-21T22:51:45.266Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "ee7acdcb-172c-47d3-85c5-6ca786d3324c"} 2025-05-21T22:51:45.266Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "ee7acdcb-172c-47d3-85c5-6ca786d3324c"} 2025-05-21T22:51:45.566Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b"} 2025-05-21T22:51:45.651Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:45.651Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:45.717Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b"} 2025-05-21T22:51:45.778Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T22:51:45.778Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "backup": "backup-aws-s3-sharded", "storage": "aws-s3"} 2025-05-21T22:51:45.778Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "backup": "backup-aws-s3-sharded", "storage": "aws-s3", "backupCmd": "backup [name: 2025-05-21T22:51:45Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T22:51:45.784Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b", "previous": "waiting", "current": "requested"} 2025-05-21T22:51:45.814Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "4cb95275-b25f-4e78-a340-055e010e966b"} 2025-05-21T22:51:45.815Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520"} 2025-05-21T22:51:45.885Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:45.885Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:45.976Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520"} 2025-05-21T22:51:45.977Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520", "backup": "backup-aws-s3-sharded", "storage": "aws-s3"} 2025-05-21T22:51:45.977Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520", "backup": "backup-aws-s3-sharded", "storage": "aws-s3", "backupCmd": "backup [name: 2025-05-21T22:51:45Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T22:51:45.983Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520", "previous": "waiting", "current": "requested"} 2025-05-21T22:51:45.999Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6fc49f7c-bb96-4c27-8220-55799520d520"} 2025-05-21T22:51:45.999Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f"} 2025-05-21T22:51:46.109Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:46.110Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:46.179Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f"} 2025-05-21T22:51:46.179Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f", "pbmName": "2025-05-21T22:51:45Z", "backup": "backup-aws-s3-sharded"} 2025-05-21T22:51:46.180Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6a6fc2f5-3a01-49d6-83ff-df3edee08e1f"} 2025-05-21T22:51:50.267Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "66c25abe-7e72-48e1-967a-e19c5c29fac1"} 2025-05-21T22:51:50.267Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-azure-blob-sharded", "reconcileID": "66c25abe-7e72-48e1-967a-e19c5c29fac1"} 2025-05-21T22:51:50.815Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3"} 2025-05-21T22:51:50.884Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:50.884Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:50.970Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3"} 2025-05-21T22:51:50.971Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"running","last_transition_ts":1747867906,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"running","iscs":true,"last_transition_ts":1747867906,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867906,"I":34},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:50.973Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3", "previous": "requested", "current": "running"} 2025-05-21T22:51:51.002Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "721723bf-642f-42a9-8cc8-53e0daa6d6a3"} 2025-05-21T22:51:51.003Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2"} 2025-05-21T22:51:51.080Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:51.080Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:51.157Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2"} 2025-05-21T22:51:51.158Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"running","last_transition_ts":1747867906,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"running","iscs":true,"last_transition_ts":1747867906,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867906,"I":34},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:51.160Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "99712e15-36fe-423a-8d22-a20ec650ebc2"} 2025-05-21T22:51:56.003Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb"} 2025-05-21T22:51:56.081Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:51:56.081Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:51:56.153Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb"} 2025-05-21T22:51:56.154Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"running","last_transition_ts":1747867906,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"running","iscs":true,"last_transition_ts":1747867906,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867911,"I":13},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:51:56.156Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7035f159-26ed-4d22-90a0-313f1c957bdb"} 2025-05-21T22:52:01.156Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b"} 2025-05-21T22:52:01.226Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:52:01.226Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:52:01.304Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b"} 2025-05-21T22:52:01.304Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"running","last_transition_ts":1747867906,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"running","iscs":true,"last_transition_ts":1747867906,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867916,"I":16},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:52:01.307Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "422ed755-3fde-4d28-9b0c-066a8976836b"} 2025-05-21T22:52:06.307Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0"} 2025-05-21T22:52:06.379Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:52:06.379Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:52:06.453Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0"} 2025-05-21T22:52:06.455Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"done","last_transition_ts":1747867925,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"},{"timestamp":1747867925,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"running","iscs":true,"last_transition_ts":1747867906,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":1133924,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867921,"I":18},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:52:06.457Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "6dc38b74-b2e5-4ba7-84f1-ac8bb5636df0"} 2025-05-21T22:52:11.458Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5"} 2025-05-21T22:52:11.537Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:52:11.537Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:52:11.616Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5"} 2025-05-21T22:52:11.617Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"done","last_transition_ts":1747867925,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"},{"timestamp":1747867925,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"done","iscs":true,"last_transition_ts":1747867929,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"},{"timestamp":1747867929,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":3703386,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867909,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867926,"I":9},"status":"running","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:52:11.622Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "13f62b15-4bb4-4ffd-b1e4-87cf54cb55f5"} 2025-05-21T22:52:16.623Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be"} 2025-05-21T22:52:16.695Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-18556"} 2025-05-21T22:52:16.695Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}} 2025-05-21T22:52:16.772Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be"} 2025-05-21T22:52:16.774Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be", "backup": "backup-aws-s3-sharded", "pbmName": "2025-05-21T22:51:45Z", "meta": {"type":"physical","opid":"682e590140f10cb361f56178","name":"2025-05-21T22:51:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747867906,"status":"done","last_transition_ts":1747867925,"first_write_ts":{"T":1747867901,"I":6},"last_write_ts":{"T":1747867901,"I":6},"node":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"},{"timestamp":1747867925,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747867906,"status":"done","iscs":true,"last_transition_ts":1747867929,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"node":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","conditions":[{"timestamp":1747867906,"status":"running"},{"timestamp":1747867929,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-physical-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":3703386,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747867906,"last_transition_ts":1747867932,"first_write_ts":{"T":1747867908,"I":1},"last_write_ts":{"T":1747867908,"I":2},"hb":{"T":1747867931,"I":3},"status":"done","conditions":[{"timestamp":1747867906,"status":"starting"},{"timestamp":1747867909,"status":"running"},{"timestamp":1747867932,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-18556.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-18556.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:52:16.775Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be", "previous": "running", "current": "ready"} 2025-05-21T22:52:16.799Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T22:52:16.830Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "7b28774b-b148-4e46-95c0-67018496a8be"} 2025-05-21T22:52:16.830Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "157da13c-2b30-47af-97ad-d416cd374b03"} 2025-05-21T22:52:16.830Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "157da13c-2b30-47af-97ad-d416cd374b03"} 2025-05-21T22:52:21.831Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2a6fd4a6-9c5f-442e-88d0-c03c651939b6"} 2025-05-21T22:52:21.831Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-physical-sharded-18556"}, "namespace": "demand-backup-physical-sharded-18556", "name": "backup-aws-s3-sharded", "reconcileID": "2a6fd4a6-9c5f-442e-88d0-c03c651939b6"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-aws-s3-sharded some-name aws-s3 s3://operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:45Z physical ready 14s 4m43s backup-azure-blob-sharded some-name azure-blob azure://operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:51:19Z physical ready 46s 4m39s backup-gcp-cs-sharded some-name gcp-cs s3://operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T22:49:03Z physical error 4m41s backup-minio-sharded some-name minio s3://operator-testing/2025-05-21T22:51:07Z physical ready 71s 4m46s Backup object psmdb-backup/backup-gcp-cs-sharded is in error state. something went wrong with operator or kubernetes cluster