Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/logs/demand-backup-incremental-sharded.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + create_infra demand-backup-incremental-sharded-20475 + local ns=demand-backup-incremental-sharded-20475 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.zfhNT4Pyv3 ++ mktemp + local LAST_ERR=/tmp/tmp.TDzdNc122q + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zfhNT4Pyv3 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.TDzdNc122q + rm /tmp/tmp.zfhNT4Pyv3 /tmp/tmp.TDzdNc122q + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-23126 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-23126 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-23126 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-23126 backup-minio-not-base-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded patched + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-23126 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.NG5UKN00qT ++ mktemp + local LAST_ERR=/tmp/tmp.wcoXnLK98E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NG5UKN00qT customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.wcoXnLK98E + rm /tmp/tmp.NG5UKN00qT /tmp/tmp.wcoXnLK98E + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Qlh5kgohGV ++ mktemp + local LAST_ERR=/tmp/tmp.uYELhrhiFp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Qlh5kgohGV + cat /tmp/tmp.uYELhrhiFp + rm /tmp/tmp.Qlh5kgohGV /tmp/tmp.uYELhrhiFp + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.rzQ779T7FU ++ mktemp + local LAST_ERR=/tmp/tmp.exaRaTZHSM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rzQ779T7FU + cat /tmp/tmp.exaRaTZHSM + rm /tmp/tmp.rzQ779T7FU /tmp/tmp.exaRaTZHSM + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.ikHq06G65Z ++ mktemp + local LAST_ERR=/tmp/tmp.fEA1VJQN36 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ikHq06G65Z clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.fEA1VJQN36 + rm /tmp/tmp.ikHq06G65Z /tmp/tmp.fEA1VJQN36 + return 0 + check_crd_for_deletion PR-1939-c5a06cd5 + local git_tag=PR-1939-c5a06cd5 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1939-c5a06cd5/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/bin/sed s/---//g + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A5D5qrzOWs +++ mktemp ++ local LAST_ERR=/tmp/tmp.zGdTcyzuuC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.A5D5qrzOWs ++ cat /tmp/tmp.zGdTcyzuuC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.A5D5qrzOWs ++ cat /tmp/tmp.zGdTcyzuuC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.A5D5qrzOWs ++ cat /tmp/tmp.zGdTcyzuuC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.A5D5qrzOWs ++ cat /tmp/tmp.zGdTcyzuuC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.A5D5qrzOWs /tmp/tmp.zGdTcyzuuC ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + local LAST_OUT=/tmp/tmp.DY621N5mWh ++ mktemp + local LAST_OUT=/tmp/tmp.jTWPfkI59F ++ mktemp + local LAST_ERR=/tmp/tmp.Nh17WEKAi4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.T3SCBWPqMZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jTWPfkI59F + cat /tmp/tmp.T3SCBWPqMZ + rm /tmp/tmp.jTWPfkI59F /tmp/tmp.T3SCBWPqMZ + return 0 namespace "demand-backup-incremental-sharded-23126" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DY621N5mWh namespace "psmdb-operator" deleted + cat /tmp/tmp.Nh17WEKAi4 + rm /tmp/tmp.DY621N5mWh /tmp/tmp.Nh17WEKAi4 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.bRJ1ruqKJR ++ mktemp + local LAST_ERR=/tmp/tmp.pVaWKbshP3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bRJ1ruqKJR + cat /tmp/tmp.pVaWKbshP3 + rm /tmp/tmp.bRJ1ruqKJR /tmp/tmp.pVaWKbshP3 + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.o56AwwwYKM ++ mktemp + local LAST_ERR=/tmp/tmp.sG5UI8uL4E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.o56AwwwYKM namespace/psmdb-operator created + cat /tmp/tmp.sG5UI8uL4E + rm /tmp/tmp.o56AwwwYKM /tmp/tmp.sG5UI8uL4E + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.G9THxoEPu1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5sZcXXm6UE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.G9THxoEPu1 ++ cat /tmp/tmp.5sZcXXm6UE ++ rm /tmp/tmp.G9THxoEPu1 /tmp/tmp.5sZcXXm6UE ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.FGWShblUYs ++ mktemp + local LAST_ERR=/tmp/tmp.T8UvSlYQDY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FGWShblUYs Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7" modified. + cat /tmp/tmp.T8UvSlYQDY + rm /tmp/tmp.FGWShblUYs /tmp/tmp.T8UvSlYQDY + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.r44BORD5E7 ++ mktemp + local LAST_ERR=/tmp/tmp.OZj4450NQx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.r44BORD5E7 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.OZj4450NQx + rm /tmp/tmp.r44BORD5E7 /tmp/tmp.OZj4450NQx + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + kubectl_bin apply -n psmdb-operator -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.A9Q7wBhPri ++ mktemp + local LAST_ERR=/tmp/tmp.i8rqJZiFeC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A9Q7wBhPri clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.i8rqJZiFeC + rm /tmp/tmp.A9Q7wBhPri /tmp/tmp.i8rqJZiFeC + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1939-c5a06cd5") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZGR1FVoHvA ++ mktemp + local LAST_ERR=/tmp/tmp.bzJDP94zSe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZGR1FVoHvA deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.bzJDP94zSe + rm /tmp/tmp.ZGR1FVoHvA /tmp/tmp.bzJDP94zSe + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.OPDKd9wMdt +++ mktemp ++ local LAST_ERR=/tmp/tmp.IKpBYnnwE4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OPDKd9wMdt ++ cat /tmp/tmp.IKpBYnnwE4 ++ rm /tmp/tmp.OPDKd9wMdt /tmp/tmp.IKpBYnnwE4 ++ return 0 + wait_pod percona-server-mongodb-operator-c7d88fc9f-bqg8j + local pod=percona-server-mongodb-operator-c7d88fc9f-bqg8j + set +o xtrace waiting for pod/percona-server-mongodb-operator-c7d88fc9f-bqg8j to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.BLb34sZYnR +++ mktemp ++ local LAST_ERR=/tmp/tmp.xOljVRFtxk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BLb34sZYnR ++ cat /tmp/tmp.xOljVRFtxk ++ rm /tmp/tmp.BLb34sZYnR /tmp/tmp.xOljVRFtxk ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-c7d88fc9f-bqg8j ++ mktemp + local LAST_OUT=/tmp/tmp.MLCCoGR6FY ++ mktemp + local LAST_ERR=/tmp/tmp.oFUVqjn8vT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-c7d88fc9f-bqg8j + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MLCCoGR6FY + cat /tmp/tmp.oFUVqjn8vT + rm /tmp/tmp.MLCCoGR6FY /tmp/tmp.oFUVqjn8vT + return 0 2025-05-21T23:13:28.139Z INFO setup Manager starting up {"gitCommit": "c5a06cd594d344bc36bde0fb5cba4f1abcbfb00f", "gitBranch": "PR-1939-c5a06cd5", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-incremental-sharded-20475 + local namespace=demand-backup-incremental-sharded-20475 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-incremental-sharded-20475' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-incremental-sharded-20475 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-incremental-sharded-20475 --ignore-not-found + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.KPomvpWxeG ++ mktemp + local LAST_ERR=/tmp/tmp.ZKRq1kavWX + local exit_status=0 + local timeout=4 ++ seq 0 2 + awk '{print$1}' + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.mGZKXFLFSi ++ mktemp + local LAST_ERR=/tmp/tmp.gCwNODbZHX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-incremental-sharded-20475 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KPomvpWxeG + cat /tmp/tmp.ZKRq1kavWX + rm /tmp/tmp.KPomvpWxeG /tmp/tmp.ZKRq1kavWX + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mGZKXFLFSi + cat /tmp/tmp.gCwNODbZHX + rm /tmp/tmp.mGZKXFLFSi /tmp/tmp.gCwNODbZHX + return 0 + kubectl_bin wait --for=delete namespace demand-backup-incremental-sharded-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.zSwgiPO0mu ++ mktemp + local LAST_ERR=/tmp/tmp.7D8Emv0f2a + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace demand-backup-incremental-sharded-20475 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zSwgiPO0mu + cat /tmp/tmp.7D8Emv0f2a + rm /tmp/tmp.zSwgiPO0mu /tmp/tmp.7D8Emv0f2a + return 0 + desc 'create namespace demand-backup-incremental-sharded-20475' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-incremental-sharded-20475 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-incremental-sharded-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.Nqco3GPs1n namespace "gmp-public" deleted ++ mktemp + local LAST_ERR=/tmp/tmp.Qs46mSiN4p + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace demand-backup-incremental-sharded-20475 namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Nqco3GPs1n namespace/demand-backup-incremental-sharded-20475 created + cat /tmp/tmp.Qs46mSiN4p + rm /tmp/tmp.Nqco3GPs1n /tmp/tmp.Qs46mSiN4p + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dp0IR45sIZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ApHHEBHnek ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Dp0IR45sIZ ++ cat /tmp/tmp.ApHHEBHnek ++ rm /tmp/tmp.Dp0IR45sIZ /tmp/tmp.ApHHEBHnek ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7 --namespace=demand-backup-incremental-sharded-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.9BJ0NhPHzi ++ mktemp + local LAST_ERR=/tmp/tmp.c9Bvn929tf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7 --namespace=demand-backup-incremental-sharded-20475 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9BJ0NhPHzi Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1939-c5a06cd5-3-cluster7" modified. + cat /tmp/tmp.c9Bvn929tf + rm /tmp/tmp.9BJ0NhPHzi /tmp/tmp.c9Bvn929tf + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Wed May 21 23:13:54 2025 NAMESPACE: demand-backup-incremental-sharded-20475 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-incremental-sharded-20475.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-incremental-sharded-20475 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-incremental-sharded-20475 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-incremental-sharded-20475 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-incremental-sharded-20475 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IldKxvzhZc +++ mktemp ++ local LAST_ERR=/tmp/tmp.EgpuQoRkEr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IldKxvzhZc ++ cat /tmp/tmp.EgpuQoRkEr ++ rm /tmp/tmp.IldKxvzhZc /tmp/tmp.EgpuQoRkEr ++ return 0 + MINIO_POD=minio-service-86dfccd949-jkdgq + wait_pod minio-service-86dfccd949-jkdgq + local pod=minio-service-86dfccd949-jkdgq + set +o xtrace waiting for pod/minio-service-86dfccd949-jkdgq to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-20475.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.hjB3LZuUPH ++ mktemp + local LAST_ERR=/tmp/tmp.Zkf32KGTDb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-20475.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hjB3LZuUPH service/minio-service created + cat /tmp/tmp.Zkf32KGTDb + rm /tmp/tmp.hjB3LZuUPH /tmp/tmp.Zkf32KGTDb + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.u6Pc5W3XyZ ++ mktemp + local LAST_ERR=/tmp/tmp.1G4ol2CNDa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u6Pc5W3XyZ make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.1G4ol2CNDa If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-20475 + rm /tmp/tmp.u6Pc5W3XyZ /tmp/tmp.1G4ol2CNDa + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AYb1hWpstt ++ mktemp + local LAST_ERR=/tmp/tmp.xFpR5OUlU4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AYb1hWpstt secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.xFpR5OUlU4 + rm /tmp/tmp.AYb1hWpstt /tmp/tmp.xFpR5OUlU4 + return 0 + desc 'Testing on sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on sharded cluster ----------------------------------------------------------------------------------- + log 'Creating PSMDB cluster' + set +o xtrace [2025-05-21T23:14:31+0000] Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.f3YRUJjw35 ++ mktemp + local LAST_ERR=/tmp/tmp.Dv9VL1IkCP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f3YRUJjw35 secret/some-users created + cat /tmp/tmp.Dv9VL1IkCP + rm /tmp/tmp.f3YRUJjw35 /tmp/tmp.Dv9VL1IkCP + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1939-c5a06cd5"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + local LAST_OUT=/tmp/tmp.7mjyS5cX3g ++ mktemp + local LAST_ERR=/tmp/tmp.arM5CDTzuO + local exit_status=0 + local timeout=4 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml + yq eval '.spec.upgradeOptions.apply="Never"' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7mjyS5cX3g perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.arM5CDTzuO + rm /tmp/tmp.7mjyS5cX3g /tmp/tmp.arM5CDTzuO + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LoHwTyShV5 ++ mktemp + local LAST_ERR=/tmp/tmp.FrlQIy0cGk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LoHwTyShV5 deployment.apps/psmdb-client created + cat /tmp/tmp.FrlQIy0cGk + rm /tmp/tmp.LoHwTyShV5 /tmp/tmp.FrlQIy0cGk + return 0 + log 'check if all pods started' + set +o xtrace [2025-05-21T23:14:36+0000] check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xkIZptGxEA +++ mktemp ++ local LAST_ERR=/tmp/tmp.7KyioqRFuj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xkIZptGxEA ++ cat /tmp/tmp.7KyioqRFuj ++ rm /tmp/tmp.xkIZptGxEA /tmp/tmp.7KyioqRFuj ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9h0K01ighN +++ mktemp ++ local LAST_ERR=/tmp/tmp.vrMopOaN7Z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9h0K01ighN ++ cat /tmp/tmp.vrMopOaN7Z ++ rm /tmp/tmp.9h0K01ighN /tmp/tmp.vrMopOaN7Z ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness...................... + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OOqWmOE2Qj +++ mktemp ++ local LAST_ERR=/tmp/tmp.W8MVe4DmZo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OOqWmOE2Qj ++ cat /tmp/tmp.W8MVe4DmZo ++ rm /tmp/tmp.OOqWmOE2Qj /tmp/tmp.W8MVe4DmZo ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WpR4bLIwUB +++ mktemp ++ local LAST_ERR=/tmp/tmp.MSsdRx26bk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WpR4bLIwUB ++ cat /tmp/tmp.MSsdRx26bk ++ rm /tmp/tmp.WpR4bLIwUB /tmp/tmp.MSsdRx26bk ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6bnajg9pud +++ mktemp ++ local LAST_ERR=/tmp/tmp.pXHvwM9wdW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6bnajg9pud ++ cat /tmp/tmp.pXHvwM9wdW ++ rm /tmp/tmp.6bnajg9pud /tmp/tmp.pXHvwM9wdW ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2Qzs1La69I +++ mktemp ++ local LAST_ERR=/tmp/tmp.DNDSW5uifN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2Qzs1La69I ++ cat /tmp/tmp.DNDSW5uifN ++ rm /tmp/tmp.2Qzs1La69I /tmp/tmp.DNDSW5uifN ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0uU68WiZEA +++ mktemp ++ local LAST_ERR=/tmp/tmp.r4doJ7X7cn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0uU68WiZEA ++ cat /tmp/tmp.r4doJ7X7cn ++ rm /tmp/tmp.0uU68WiZEA /tmp/tmp.r4doJ7X7cn ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo ++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}' ++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lL1I9ou6YF +++ mktemp ++ local LAST_ERR=/tmp/tmp.4HA2E3TzGl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get svc some-name-mongos '-o=jsonpath={.status}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lL1I9ou6YF ++ cat /tmp/tmp.4HA2E3TzGl ++ rm /tmp/tmp.lL1I9ou6YF /tmp/tmp.4HA2E3TzGl ++ return 0 + lbEndpoint=34.59.39.183 + '[' -z 34.59.39.183 ']' + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-20475 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-20475 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LIX1WhtCOH +++ mktemp ++ local LAST_ERR=/tmp/tmp.jGTvtcOkdX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LIX1WhtCOH ++ cat /tmp/tmp.jGTvtcOkdX ++ rm /tmp/tmp.LIX1WhtCOH /tmp/tmp.jGTvtcOkdX ++ return 0 + local client_container=psmdb-client-b9788d8bc-czdqk + kubectl_bin exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.7poEHI5JWv ++ mktemp + local LAST_ERR=/tmp/tmp.SvYVMGUOGf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7poEHI5JWv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("cf3c7ca4-9cf9-46ad-aa95-235707a4b293") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.SvYVMGUOGf + rm /tmp/tmp.7poEHI5JWv /tmp/tmp.SvYVMGUOGf + return 0 + sleep 1 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G6Em60SwFi +++ mktemp ++ local LAST_ERR=/tmp/tmp.TbWZiUl8dx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.G6Em60SwFi ++ cat /tmp/tmp.TbWZiUl8dx ++ rm /tmp/tmp.G6Em60SwFi /tmp/tmp.TbWZiUl8dx ++ return 0 + local client_container=psmdb-client-b9788d8bc-czdqk + kubectl_bin exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.24fKR3eK4G ++ mktemp + local LAST_ERR=/tmp/tmp.n02rMEVnat + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.24fKR3eK4G Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("17a6d37e-fec9-4f66-8604-caa6ac5e90f3") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.n02rMEVnat + rm /tmp/tmp.24fKR3eK4G /tmp/tmp.n02rMEVnat + return 0 + sleep 5 + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 -sharded + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 + local postfix=-sharded + local suffix= + local database=myApp + local collection=test + local port=27017 + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V1vjkwNbuf +++ mktemp ++ local LAST_ERR=/tmp/tmp.FJi2Y3zSGa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V1vjkwNbuf ++ cat /tmp/tmp.FJi2Y3zSGa ++ rm /tmp/tmp.V1vjkwNbuf /tmp/tmp.FJi2Y3zSGa ++ return 0 + local client_container=psmdb-client-b9788d8bc-czdqk + kubectl_bin exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.GoBvfOwn1T ++ mktemp + local LAST_ERR=/tmp/tmp.yuKwDhf0cz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-b9788d8bc-czdqk -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-20475.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GoBvfOwn1T + cat /tmp/tmp.yuKwDhf0cz + rm /tmp/tmp.GoBvfOwn1T /tmp/tmp.yuKwDhf0cz + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.HN9XvIXRCh/find-sharded + log 'waiting 60 seconds for stable timestamp in wiredtiger' + set +o xtrace [2025-05-21T23:18:00+0000] waiting 60 seconds for stable timestamp in wiredtiger + sleep 80 + log 'running backups' + set +o xtrace [2025-05-21T23:19:20+0000] running backups + '[' -z '' ']' + backup_name_aws=backup-aws-s3-sharded + backup_name_gcp=backup-gcp-cs-sharded + backup_name_azure=backup-azure-blob-sharded + run_backup aws-s3 backup-aws-s3-sharded + local storage=aws-s3 + local backup_name=backup-aws-s3-sharded + local base=true + local backup_type=incremental + [[ true == \t\r\u\e ]] + backup_type=incremental-base + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml + /usr/bin/sed -e 's/storageName:/storageName: aws-s3/' + kubectl_bin apply -f - + yq '.spec.type="incremental-base"' ++ mktemp + local LAST_OUT=/tmp/tmp.T2bZmKFkCn + /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/' ++ mktemp + local LAST_ERR=/tmp/tmp.BuyCY2uVl8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T2bZmKFkCn perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created + cat /tmp/tmp.BuyCY2uVl8 + rm /tmp/tmp.T2bZmKFkCn /tmp/tmp.BuyCY2uVl8 + return 0 + run_backup gcp-cs backup-gcp-cs-sharded + local storage=gcp-cs + local backup_name=backup-gcp-cs-sharded + local base=true + local backup_type=incremental + [[ true == \t\r\u\e ]] + backup_type=incremental-base + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml + /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/' + kubectl_bin apply -f - + /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/' + yq '.spec.type="incremental-base"' ++ mktemp + local LAST_OUT=/tmp/tmp.FdvXXdlzqO ++ mktemp + local LAST_ERR=/tmp/tmp.znoQh4hXUZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FdvXXdlzqO perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created + cat /tmp/tmp.znoQh4hXUZ + rm /tmp/tmp.FdvXXdlzqO /tmp/tmp.znoQh4hXUZ + return 0 + run_backup azure-blob backup-azure-blob-sharded + local storage=azure-blob + local backup_name=backup-azure-blob-sharded + local base=true + local backup_type=incremental + [[ true == \t\r\u\e ]] + backup_type=incremental-base + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml + /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/' + yq '.spec.type="incremental-base"' + kubectl_bin apply -f - ++ mktemp + /usr/bin/sed -e 's/storageName:/storageName: azure-blob/' + local LAST_OUT=/tmp/tmp.to3SKV93Pf ++ mktemp + local LAST_ERR=/tmp/tmp.h9bZL876yR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.to3SKV93Pf perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created + cat /tmp/tmp.h9bZL876yR + rm /tmp/tmp.to3SKV93Pf /tmp/tmp.h9bZL876yR + return 0 + wait_backup backup-aws-s3-sharded + local backup_name=backup-aws-s3-sharded + local target_state=ready + set +o xtrace waiting for backup-aws-s3-sharded to reach ready state.......... + check_backup_in_storage backup-aws-s3-sharded s3 rs0 + local backup=backup-aws-s3-sharded + local storage_type=s3 + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=s3.amazonaws.com ++ get_backup_dest backup-aws-s3-sharded ++ local backup_name=backup-aws-s3-sharded ++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ sed 's|azure://||' ++ sed 's|s3://||' ++ local LAST_OUT=/tmp/tmp.tximkbcHRl ++ sed -e 's/.json$//' +++ mktemp ++ local LAST_ERR=/tmp/tmp.IVKCdnyi0B ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tximkbcHRl ++ cat /tmp/tmp.IVKCdnyi0B ++ rm /tmp/tmp.tximkbcHRl /tmp/tmp.IVKCdnyi0B ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z + local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/rs0/filelist.pbm + log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/rs0/filelist.pbm exists' + set +o xtrace [2025-05-21T23:19:50+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/rs0/filelist.pbm exists + curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 10127 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK x-amz-id-2: aBmwEn3jpCfdBPi82NmnJUcV355DqGNkqk9nJDqD7kbK5p5SZ52Z8zKCquAA2aeCHhnwBBLzXhI= x-amz-request-id: BGFVBDMNZ5406G0E Date: Wed, 21 May 2025 23:19:52 GMT Last-Modified: Wed, 21 May 2025 23:19:41 GMT x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup" ETag: "18b5f9035837fcf7ce28f885510cb00d" x-amz-server-side-encryption: AES256 Accept-Ranges: bytes Content-Type: binary/octet-stream Content-Length: 10127 Server: AmazonS3 + check_backup_in_storage backup-aws-s3-sharded s3 cfg + local backup=backup-aws-s3-sharded + local storage_type=s3 + local replset=cfg + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=s3.amazonaws.com ++ get_backup_dest backup-aws-s3-sharded ++ local backup_name=backup-aws-s3-sharded ++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QDTX6qGcRK +++ mktemp ++ local LAST_ERR=/tmp/tmp.P9D7MvWkbG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QDTX6qGcRK ++ cat /tmp/tmp.P9D7MvWkbG ++ rm /tmp/tmp.QDTX6qGcRK /tmp/tmp.P9D7MvWkbG ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z + local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/cfg/filelist.pbm + log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/cfg/filelist.pbm exists' + set +o xtrace [2025-05-21T23:19:52+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/cfg/filelist.pbm exists + curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z/cfg/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 18299 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 18299 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK x-amz-id-2: F4ESM8jzzDd+mGBmCuf3TwRo34n7q4MWoqU6BPrGQho04MDBOqqU0mfE6PRB8RLL7UudFGkjIsE= x-amz-request-id: QTVCKF0XX4C78BDW Date: Wed, 21 May 2025 23:19:53 GMT Last-Modified: Wed, 21 May 2025 23:19:47 GMT x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup" ETag: "007f5430d104f1128e82a10e15de96ec" x-amz-server-side-encryption: AES256 Accept-Ranges: bytes Content-Type: binary/octet-stream Content-Length: 18299 Server: AmazonS3 + wait_backup backup-gcp-cs-sharded + local backup_name=backup-gcp-cs-sharded + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sharded to reach ready state......... + check_backup_in_storage backup-gcp-cs-sharded gcs rs0 + local backup=backup-gcp-cs-sharded + local storage_type=gcs + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=storage.googleapis.com ++ get_backup_dest backup-gcp-cs-sharded ++ local backup_name=backup-gcp-cs-sharded ++ sed 's|azure://||' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JtwmZGdmYj +++ mktemp ++ local LAST_ERR=/tmp/tmp.K4mXjY7feu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JtwmZGdmYj ++ cat /tmp/tmp.K4mXjY7feu ++ rm /tmp/tmp.JtwmZGdmYj /tmp/tmp.K4mXjY7feu ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z + local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/rs0/filelist.pbm + log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/rs0/filelist.pbm exists' + set +o xtrace [2025-05-21T23:20:15+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/rs0/filelist.pbm exists + curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 10127 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/2 200 content-type: application/octet-stream x-guploader-uploadid: AAO2VwpCa7IwNWQaccCXWu3yhb5MY1JSJQrs8bkXVajidFv28bb9ZetZvujHwIsUpv1mPIM expires: Thu, 22 May 2025 00:20:15 GMT date: Wed, 21 May 2025 23:20:15 GMT cache-control: public, max-age=3600 last-modified: Wed, 21 May 2025 23:20:04 GMT etag: "2d371ee48fccf5f1ef8b3117b53f6988" x-goog-generation: 1747869604082559 x-goog-metageneration: 1 x-goog-stored-content-encoding: identity x-goog-stored-content-length: 10127 x-goog-hash: crc32c=gtKqgA== x-goog-hash: md5=LTce5I/M9fHvizEXtT9piA== x-goog-expiration: Thu, 22 May 2025 23:20:04 GMT x-goog-storage-class: STANDARD accept-ranges: bytes content-length: 10127 server: UploadServer alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + check_backup_in_storage backup-gcp-cs-sharded gcs cfg + local backup=backup-gcp-cs-sharded + local storage_type=gcs + local replset=cfg + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=storage.googleapis.com ++ get_backup_dest backup-gcp-cs-sharded ++ local backup_name=backup-gcp-cs-sharded ++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ sed 's|azure://||' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ local LAST_OUT=/tmp/tmp.qhN3XlZKgk +++ mktemp ++ local LAST_ERR=/tmp/tmp.U7zaPhOctS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qhN3XlZKgk ++ cat /tmp/tmp.U7zaPhOctS ++ rm /tmp/tmp.qhN3XlZKgk /tmp/tmp.U7zaPhOctS ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z + local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/cfg/filelist.pbm + log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/cfg/filelist.pbm exists' + set +o xtrace [2025-05-21T23:20:16+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/cfg/filelist.pbm exists + curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z/cfg/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 18299 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/2 200 content-type: application/octet-stream x-guploader-uploadid: AAO2Vwq2e6Lsd1yTqoPmEMAlRcxRHXvMjk27c9pHipFbBlhRuKf3dgJL6mpfnRCh53-84fZh expires: Thu, 22 May 2025 00:20:17 GMT date: Wed, 21 May 2025 23:20:17 GMT cache-control: public, max-age=3600 last-modified: Wed, 21 May 2025 23:20:08 GMT etag: "0b691263d07ed39692f5bc3023eae872" x-goog-generation: 1747869608703531 x-goog-metageneration: 1 x-goog-stored-content-encoding: identity x-goog-stored-content-length: 18299 x-goog-hash: crc32c=wp7iQg== x-goog-hash: md5=C2kSY9B+05aS9bwwI+rocg== x-goog-expiration: Thu, 22 May 2025 23:20:08 GMT x-goog-storage-class: STANDARD accept-ranges: bytes content-length: 18299 server: UploadServer alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + wait_backup backup-azure-blob-sharded + local backup_name=backup-azure-blob-sharded + local target_state=ready + set +o xtrace waiting for backup-azure-blob-sharded to reach ready state......... + check_backup_in_storage backup-azure-blob-sharded azure rs0 + local backup=backup-azure-blob-sharded + local storage_type=azure + local replset=rs0 + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=engk8soperators.blob.core.windows.net ++ get_backup_dest backup-azure-blob-sharded ++ local backup_name=backup-azure-blob-sharded ++ sed 's|azure://||' ++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GGKQx1E7zk +++ mktemp ++ local LAST_ERR=/tmp/tmp.IKIz9vJubO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GGKQx1E7zk ++ cat /tmp/tmp.IKIz9vJubO ++ rm /tmp/tmp.GGKQx1E7zk /tmp/tmp.IKIz9vJubO ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z + local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/rs0/filelist.pbm + log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/rs0/filelist.pbm exists' + set +o xtrace [2025-05-21T23:20:39+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/rs0/filelist.pbm exists + curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/rs0/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 12833 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK Content-Length: 12833 Content-Type: application/octet-stream Content-MD5: /xFjyXQ2ep2YB3+cOTlkmw== Last-Modified: Wed, 21 May 2025 23:20:29 GMT ETag: 0x8DD98BE13BA9DE0 Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-request-id: 914236ce-201e-00a7-64a6-ca37ad000000 x-ms-version: 2009-09-19 x-ms-lease-status: unlocked x-ms-blob-type: BlockBlob Date: Wed, 21 May 2025 23:20:38 GMT + check_backup_in_storage backup-azure-blob-sharded azure cfg + local backup=backup-azure-blob-sharded + local storage_type=azure + local replset=cfg + local file=filelist.pbm + local endpoint + case ${storage_type} in + endpoint=engk8soperators.blob.core.windows.net ++ get_backup_dest backup-azure-blob-sharded ++ local backup_name=backup-azure-blob-sharded ++ sed 's|azure://||' ++ sed 's|s3://||' ++ sed -e 's/.json$//' ++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SFV2X9Nn7e +++ mktemp ++ local LAST_ERR=/tmp/tmp.F6oGfA40t0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SFV2X9Nn7e ++ cat /tmp/tmp.F6oGfA40t0 ++ rm /tmp/tmp.SFV2X9Nn7e /tmp/tmp.F6oGfA40t0 ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z + local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/cfg/filelist.pbm + log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/cfg/filelist.pbm exists' + set +o xtrace [2025-05-21T23:20:41+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/cfg/filelist.pbm exists + curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z/cfg/filelist.pbm % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 19099 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 HTTP/1.1 200 OK Content-Length: 19099 Content-Type: application/octet-stream Content-MD5: nnfRgiQZurnrVB73HvqJ7A== Last-Modified: Wed, 21 May 2025 23:20:33 GMT ETag: 0x8DD98BE15ABC589 Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-request-id: 97b5c2bf-c01e-004b-5da6-ca233c000000 x-ms-version: 2009-09-19 x-ms-lease-status: unlocked x-ms-blob-type: BlockBlob Date: Wed, 21 May 2025 23:20:41 GMT + backup_name_minio=backup-minio-sharded + run_backup minio backup-minio-sharded + local storage=minio + local backup_name=backup-minio-sharded + local base=true + local backup_type=incremental + [[ true == \t\r\u\e ]] + backup_type=incremental-base + /usr/bin/sed -e 's/storageName:/storageName: minio/' + /usr/bin/sed -e 's/name:/name: backup-minio-sharded/' + yq '.spec.type="incremental-base"' + kubectl_bin apply -f - + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1939/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.OcggxgIwK0 ++ mktemp + local LAST_ERR=/tmp/tmp.XacPohML0p + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OcggxgIwK0 perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created + cat /tmp/tmp.XacPohML0p + rm /tmp/tmp.OcggxgIwK0 /tmp/tmp.XacPohML0p + return 0 + wait_backup backup-minio-sharded + local backup_name=backup-minio-sharded + local target_state=ready + set +o xtrace waiting for backup-minio-sharded to reach ready state........................................................2025-05-21T23:20:24.992Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "75695397-440f-4914-9eb5-93db369793f5", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:24.992Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "75695397-440f-4914-9eb5-93db369793f5", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T23:20:25.062Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "75695397-440f-4914-9eb5-93db369793f5"} 2025-05-21T23:20:25.063Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "75695397-440f-4914-9eb5-93db369793f5", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T23:20:14Z", "meta": {"type":"incremental","opid":"682e5fae66ec5adde5c3a28c","name":"2025-05-21T23:20:14Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747869615,"status":"running","last_transition_ts":1747869615,"first_write_ts":{"T":1747869609,"I":2},"last_write_ts":{"T":1747869609,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747869615,"status":"running","iscs":true,"last_transition_ts":1747869615,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"node":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-incremental-sharded"}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869615,"last_transition_ts":1747869618,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"hb":{"T":1747869620,"I":19},"status":"running","conditions":[{"timestamp":1747869615,"status":"starting"},{"timestamp":1747869618,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:25.066Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "75695397-440f-4914-9eb5-93db369793f5"} 2025-05-21T23:20:30.066Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8"} 2025-05-21T23:20:30.131Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:30.131Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T23:20:30.200Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8"} 2025-05-21T23:20:30.201Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T23:20:14Z", "meta": {"type":"incremental","opid":"682e5fae66ec5adde5c3a28c","name":"2025-05-21T23:20:14Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747869615,"status":"done","last_transition_ts":1747869629,"first_write_ts":{"T":1747869609,"I":2},"last_write_ts":{"T":1747869609,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"},{"timestamp":1747869629,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747869615,"status":"running","iscs":true,"last_transition_ts":1747869615,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"node":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-incremental-sharded"}},"size":1001604,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869615,"last_transition_ts":1747869618,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"hb":{"T":1747869625,"I":30},"status":"running","conditions":[{"timestamp":1747869615,"status":"starting"},{"timestamp":1747869618,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:30.203Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "8bafc31c-1bac-4914-a54b-a77b04f8f4b8"} 2025-05-21T23:20:35.203Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b"} 2025-05-21T23:20:35.279Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:35.279Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}} 2025-05-21T23:20:35.366Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b"} 2025-05-21T23:20:35.367Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b", "backup": "backup-azure-blob-sharded", "pbmName": "2025-05-21T23:20:14Z", "meta": {"type":"incremental","opid":"682e5fae66ec5adde5c3a28c","name":"2025-05-21T23:20:14Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","start_ts":1747869615,"status":"done","last_transition_ts":1747869629,"first_write_ts":{"T":1747869609,"I":2},"last_write_ts":{"T":1747869609,"I":2},"node":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"},{"timestamp":1747869629,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"shardsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","start_ts":1747869615,"status":"done","iscs":true,"last_transition_ts":1747869633,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"node":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","conditions":[{"timestamp":1747869615,"status":"running"},{"timestamp":1747869633,"status":"done"}],"mongod_opts":{"net":{"bindIp":"*","port":27017},"sharding":{"clusterRole":"configsvr"},"storage":{"directoryPerDB":false,"dbPath":"/data/db","wiredTiger":{"engineConfig":{"journalCompressor":"snappy","directoryForIndexes":false},"collectionConfig":{"blockCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"security":{"enableEncryption":true,"encryptionKeyFile":"/etc/mongodb-encryption/encryption-key","relaxPermChecks":true}},"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"azure-blob","profile":true,"type":"azure","azure":{"account":"engk8soperators","container":"operator-testing","prefix":"psmdb-demand-backup-incremental-sharded"}},"size":3204787,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869615,"last_transition_ts":1747869635,"first_write_ts":{"T":1747869617,"I":2},"last_write_ts":{"T":1747869617,"I":3},"hb":{"T":1747869630,"I":16},"status":"done","conditions":[{"timestamp":1747869615,"status":"starting"},{"timestamp":1747869618,"status":"running"},{"timestamp":1747869635,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:35.370Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b", "previous": "running", "current": "ready"} 2025-05-21T23:20:35.396Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T23:20:35.421Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "6306e563-04d4-4bf4-bd7e-6de4a035525b"} 2025-05-21T23:20:35.421Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "a171284f-af92-4e11-9f5a-3123bdbe9c8f"} 2025-05-21T23:20:35.421Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "a171284f-af92-4e11-9f5a-3123bdbe9c8f"} 2025-05-21T23:20:40.422Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "a4c2cbe6-725e-4c55-a677-26eb411f026a"} 2025-05-21T23:20:40.422Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-azure-blob-sharded", "reconcileID": "a4c2cbe6-725e-4c55-a677-26eb411f026a"} 2025-05-21T23:20:43.742Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db"} 2025-05-21T23:20:43.818Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:43.818Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:43.898Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db"} 2025-05-21T23:20:43.933Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T23:20:43.933Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "backup": "backup-minio-sharded", "storage": "minio"} 2025-05-21T23:20:43.933Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "backup": "backup-minio-sharded", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T23:20:43Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T23:20:43.940Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db", "previous": "", "current": "requested"} 2025-05-21T23:20:43.962Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "2c8a4280-329a-4c3b-90a1-2aa3b0a425db"} 2025-05-21T23:20:43.962Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f"} 2025-05-21T23:20:44.045Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:44.045Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:44.113Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f"} 2025-05-21T23:20:44.113Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f", "backup": "backup-minio-sharded", "storage": "minio"} 2025-05-21T23:20:44.114Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f", "backup": "backup-minio-sharded", "storage": "minio", "backupCmd": "backup [name: 2025-05-21T23:20:44Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-21T23:20:44.119Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f", "previous": "", "current": "requested"} 2025-05-21T23:20:44.152Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "14a88d91-5c6c-4607-9f2e-40e69b70308f"} 2025-05-21T23:20:44.152Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5"} 2025-05-21T23:20:44.234Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:44.234Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:44.306Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5"} 2025-05-21T23:20:44.307Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5", "pbmName": "2025-05-21T23:20:43Z", "backup": "backup-minio-sharded"} 2025-05-21T23:20:44.308Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "f40276c5-f6a7-4ab8-af70-5e788adf9ce5"} 2025-05-21T23:20:44.309Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c"} 2025-05-21T23:20:44.383Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:44.383Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:44.494Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c"} 2025-05-21T23:20:44.496Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c", "pbmName": "2025-05-21T23:20:44Z", "backup": "backup-minio-sharded"} 2025-05-21T23:20:44.498Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3b8914fe-db36-48f9-96a7-bb0a12d47a2c"} 2025-05-21T23:20:48.962Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142"} 2025-05-21T23:20:49.033Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:49.034Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:49.101Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142"} 2025-05-21T23:20:49.101Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869644,"I":45},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:49.104Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "dd129f35-e9e6-409d-9e65-2aa991df9142"} 2025-05-21T23:20:54.104Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f"} 2025-05-21T23:20:54.175Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:54.175Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:54.252Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f"} 2025-05-21T23:20:54.253Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:54.255Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6c764c50-e3c5-44fe-8662-7c7d75c2494f"} 2025-05-21T23:20:59.256Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d"} 2025-05-21T23:20:59.322Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:20:59.322Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:20:59.394Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d"} 2025-05-21T23:20:59.396Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:20:59.397Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "759ddcec-fe24-4fbf-89dc-a88bad9e4d0d"} 2025-05-21T23:21:04.398Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07"} 2025-05-21T23:21:04.485Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:04.485Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:04.555Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07"} 2025-05-21T23:21:04.556Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:04.557Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3c4e727d-2a1f-49a9-98d6-4215048eed07"} 2025-05-21T23:21:09.558Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e"} 2025-05-21T23:21:09.627Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:09.627Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:09.707Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e"} 2025-05-21T23:21:09.709Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:09.712Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "1044e889-449a-4f5f-9b7b-6836cf33794e"} 2025-05-21T23:21:14.712Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d"} 2025-05-21T23:21:14.782Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:14.782Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:14.853Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d"} 2025-05-21T23:21:14.854Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:14.856Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "d643b1af-cb65-40c8-8af1-193735cbce3d"} 2025-05-21T23:21:19.856Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc"} 2025-05-21T23:21:19.933Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:19.933Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:19.998Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc"} 2025-05-21T23:21:19.998Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:20.000Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "b2dff625-1f03-45c6-9608-2222cc3741fc"} 2025-05-21T23:21:25.001Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3"} 2025-05-21T23:21:25.067Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:25.067Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:25.131Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3"} 2025-05-21T23:21:25.131Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:25.133Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "514f0eb1-cbeb-468b-9991-b3072e77d5e3"} 2025-05-21T23:21:30.134Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc"} 2025-05-21T23:21:30.200Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:30.200Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:30.265Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc"} 2025-05-21T23:21:30.266Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:30.268Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "53ace216-b289-40e7-81d2-a97f5c0f2edc"} 2025-05-21T23:21:35.268Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb"} 2025-05-21T23:21:35.336Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:35.336Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:35.407Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb"} 2025-05-21T23:21:35.408Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:35.410Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "615622c8-ac16-4bd2-86d1-6e329f0d12cb"} 2025-05-21T23:21:40.410Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb"} 2025-05-21T23:21:40.498Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:40.498Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:40.565Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb"} 2025-05-21T23:21:40.566Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:40.567Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "fe09290c-d5a8-4ff0-a9cf-061c5433d1bb"} 2025-05-21T23:21:45.569Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6"} 2025-05-21T23:21:45.642Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:45.642Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:45.725Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6"} 2025-05-21T23:21:45.726Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:45.728Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3bd88e47-768a-4caf-88a5-2b6a0accdcf6"} 2025-05-21T23:21:50.728Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346"} 2025-05-21T23:21:50.796Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:50.796Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:50.869Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346"} 2025-05-21T23:21:50.870Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:50.872Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "151a3e7a-a8ab-4526-bec7-07aa26cae346"} 2025-05-21T23:21:55.874Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397"} 2025-05-21T23:21:55.953Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:21:55.953Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:21:56.030Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397"} 2025-05-21T23:21:56.031Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:21:56.033Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "74350ef3-c659-4f60-b470-4454c4d67397"} 2025-05-21T23:22:01.034Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0"} 2025-05-21T23:22:01.104Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:01.104Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:01.172Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0"} 2025-05-21T23:22:01.173Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:01.175Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "770b8901-e055-4a58-9e5b-f727534dd8b0"} 2025-05-21T23:22:06.176Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8"} 2025-05-21T23:22:06.244Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:06.244Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:06.313Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8"} 2025-05-21T23:22:06.314Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:06.315Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "310aacc8-06f2-4bc1-a8c5-aed6bfe306e8"} 2025-05-21T23:22:11.316Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de"} 2025-05-21T23:22:11.406Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:11.406Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:11.510Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de"} 2025-05-21T23:22:11.511Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:11.513Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "3689d6dc-db80-4555-ab02-d949f1dbc7de"} 2025-05-21T23:22:16.514Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33"} 2025-05-21T23:22:16.578Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:16.578Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:16.640Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33"} 2025-05-21T23:22:16.641Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:16.643Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "28d01137-07be-4f4c-aefd-fb213350df33"} 2025-05-21T23:22:21.643Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6"} 2025-05-21T23:22:21.729Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:21.729Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:21.804Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6"} 2025-05-21T23:22:21.805Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:21.806Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8c0ad75c-7558-4abd-9904-41c287a0e2d6"} 2025-05-21T23:22:26.807Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88"} 2025-05-21T23:22:26.878Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:26.878Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:26.952Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88"} 2025-05-21T23:22:26.954Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:26.955Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "6b970004-5e00-4971-8bc0-7856349a5d88"} 2025-05-21T23:22:31.956Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c"} 2025-05-21T23:22:32.030Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:32.030Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:32.101Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c"} 2025-05-21T23:22:32.102Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:32.104Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "31966023-b254-44ff-9e5d-0f2f2051cf6c"} 2025-05-21T23:22:37.104Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110"} 2025-05-21T23:22:37.176Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:37.176Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:37.249Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110"} 2025-05-21T23:22:37.250Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:37.252Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "8f815bee-a724-4bda-8ff9-5f807f06e110"} 2025-05-21T23:22:42.253Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76"} 2025-05-21T23:22:42.329Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:42.329Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:42.411Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76"} 2025-05-21T23:22:42.412Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:42.415Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "9610b57a-e5c9-421c-91a0-46a2500ebc76"} 2025-05-21T23:22:47.416Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410"} 2025-05-21T23:22:47.502Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-20475"} 2025-05-21T23:22:47.502Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410", "currentJob": {"Name":"backup-minio-sharded","Type":0}} 2025-05-21T23:22:47.567Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410"} 2025-05-21T23:22:47.569Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410", "backup": "backup-minio-sharded", "pbmName": "2025-05-21T23:20:44Z", "meta": {"type":"incremental","opid":"682e5fcc66ec5adde5c3a2cb","name":"2025-05-21T23:20:44Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747869644,"last_transition_ts":1747869644,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747869649,"I":51},"status":"starting","conditions":[{"timestamp":1747869644,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.demand-backup-incremental-sharded-20475.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T23:22:47.570Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410", "previous": "requested", "current": "error"} 2025-05-21T23:22:47.597Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T23:22:47.623Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "bfb0c887-d3d8-4f6a-8b17-dc01a91e1410"} 2025-05-21T23:22:47.623Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "e8f38d72-5144-482f-bf09-e5def280b1cb"} 2025-05-21T23:22:47.623Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-sharded","namespace":"demand-backup-incremental-sharded-20475"}, "namespace": "demand-backup-incremental-sharded-20475", "name": "backup-minio-sharded", "reconcileID": "e8f38d72-5144-482f-bf09-e5def280b1cb"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-aws-s3-sharded some-name aws-s3 s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:24Z incremental-base 1.84MB ready 3m3s 3m28s backup-azure-blob-sharded some-name azure-blob azure://operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:20:14Z incremental-base 3.06MB ready 2m16s 3m23s backup-gcp-cs-sharded some-name gcp-cs s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-21T23:19:51Z incremental-base 2.23MB ready 2m41s 3m26s backup-minio-sharded some-name minio s3://operator-testing/2025-05-21T23:20:44Z incremental-base error 2m8s Backup object psmdb-backup/backup-minio-sharded is in error state. something went wrong with operator or kubernetes cluster