++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/logs/demand-backup-sharded.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/logs/demand-backup-sharded.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep -eks- Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep gke Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ '[' v1.32.11-gke.1174000 ']' ++ GKE=1 +++ kubectl version -o json +++ /usr/sbin/sed -r 's/[^0-9.]+//g' +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.32 + set_debug + [[ 1 == 1 ]] + set -o xtrace + custom_port=27019 + create_infra demand-backup-sharded-7357 + local ns=demand-backup-sharded-7357 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.PFalIu9L9z ++ mktemp + local LAST_ERR=/tmp/tmp.Z26RUy4zKK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PFalIu9L9z customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.Z26RUy4zKK + rm /tmp/tmp.PFalIu9L9z /tmp/tmp.Z26RUy4zKK + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JyOfhH6Bux ++ mktemp + local LAST_ERR=/tmp/tmp.vFYl7Bt68M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JyOfhH6Bux + cat /tmp/tmp.vFYl7Bt68M + rm /tmp/tmp.JyOfhH6Bux /tmp/tmp.vFYl7Bt68M + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.cE1FmjQuMq ++ mktemp + local LAST_ERR=/tmp/tmp.mPztgh3TlU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cE1FmjQuMq + cat /tmp/tmp.mPztgh3TlU + rm /tmp/tmp.cE1FmjQuMq /tmp/tmp.mPztgh3TlU + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.aIrbeMYA37 ++ mktemp + local LAST_ERR=/tmp/tmp.nyRkRSQyvE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aIrbeMYA37 + cat /tmp/tmp.nyRkRSQyvE + rm /tmp/tmp.aIrbeMYA37 /tmp/tmp.nyRkRSQyvE + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.YiPcwOZK3k ++ mktemp + local LAST_ERR=/tmp/tmp.feR1hNAQiS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YiPcwOZK3k clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.feR1hNAQiS + rm /tmp/tmp.YiPcwOZK3k /tmp/tmp.feR1hNAQiS + return 0 + check_crd_for_deletion PR-2226-1e37b178 + local git_tag=PR-2226-1e37b178 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2226-1e37b178/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GJIzNZJo3I +++ mktemp ++ local LAST_ERR=/tmp/tmp.DOXPxtkIUt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GJIzNZJo3I ++ cat /tmp/tmp.DOXPxtkIUt Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GJIzNZJo3I ++ cat /tmp/tmp.DOXPxtkIUt Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.GJIzNZJo3I ++ cat /tmp/tmp.DOXPxtkIUt Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.GJIzNZJo3I ++ cat /tmp/tmp.DOXPxtkIUt Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.GJIzNZJo3I /tmp/tmp.DOXPxtkIUt ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.whTV7MV2WU + local LAST_OUT=/tmp/tmp.LlZiBRYl73 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.jn2DauRe9S + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.bdSGYhGhWN + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LlZiBRYl73 + cat /tmp/tmp.bdSGYhGhWN + rm /tmp/tmp.LlZiBRYl73 /tmp/tmp.bdSGYhGhWN + return 0 namespace "demand-backup-sharded-24619" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.whTV7MV2WU namespace "psmdb-operator" deleted + cat /tmp/tmp.jn2DauRe9S + rm /tmp/tmp.whTV7MV2WU /tmp/tmp.jn2DauRe9S + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.vYthDwmuX9 ++ mktemp + local LAST_ERR=/tmp/tmp.Cr9DPWMIeh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vYthDwmuX9 + cat /tmp/tmp.Cr9DPWMIeh + rm /tmp/tmp.vYthDwmuX9 /tmp/tmp.Cr9DPWMIeh + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.TkIgMQyKRe ++ mktemp + local LAST_ERR=/tmp/tmp.Mrh1pyxvKz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TkIgMQyKRe namespace/psmdb-operator created + cat /tmp/tmp.Mrh1pyxvKz + rm /tmp/tmp.TkIgMQyKRe /tmp/tmp.Mrh1pyxvKz + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.DXoKUF7LxJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.opltPqzqGh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DXoKUF7LxJ ++ cat /tmp/tmp.opltPqzqGh ++ rm /tmp/tmp.DXoKUF7LxJ /tmp/tmp.opltPqzqGh ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7EvYwYlovc ++ mktemp + local LAST_ERR=/tmp/tmp.CoDbp6RnVF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7EvYwYlovc Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4" modified. + cat /tmp/tmp.CoDbp6RnVF + rm /tmp/tmp.7EvYwYlovc /tmp/tmp.CoDbp6RnVF + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2226-1e37b178' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2226-1e37b178 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.F2HCDdNug1 ++ mktemp + local LAST_ERR=/tmp/tmp.kx94eqR4R7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F2HCDdNug1 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.kx94eqR4R7 + rm /tmp/tmp.F2HCDdNug1 /tmp/tmp.kx94eqR4R7 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.bH85JV50q4 ++ mktemp + local LAST_ERR=/tmp/tmp.XfoJvo1drD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bH85JV50q4 clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.XfoJvo1drD + rm /tmp/tmp.bH85JV50q4 /tmp/tmp.XfoJvo1drD + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2226-1e37b178") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Z3KicTpwpq ++ mktemp + local LAST_ERR=/tmp/tmp.njNJVXTnWl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z3KicTpwpq deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.njNJVXTnWl + rm /tmp/tmp.Z3KicTpwpq /tmp/tmp.njNJVXTnWl + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.emNI00dyQ9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.coMCD7btEd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.emNI00dyQ9 ++ cat /tmp/tmp.coMCD7btEd ++ rm /tmp/tmp.emNI00dyQ9 /tmp/tmp.coMCD7btEd ++ return 0 + wait_operator_pod percona-server-mongodb-operator-bff7975f5-4x9tn + local pod=percona-server-mongodb-operator-bff7975f5-4x9tn + set +o xtrace waiting for pod/percona-server-mongodb-operator-bff7975f5-4x9tn to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.FY69t8RYW8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4WQRw4yfo2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FY69t8RYW8 ++ cat /tmp/tmp.4WQRw4yfo2 ++ rm /tmp/tmp.FY69t8RYW8 /tmp/tmp.4WQRw4yfo2 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-bff7975f5-4x9tn ++ mktemp + local LAST_OUT=/tmp/tmp.2DoFfom3wP ++ mktemp + local LAST_ERR=/tmp/tmp.hkmvt8obt2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-bff7975f5-4x9tn + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2DoFfom3wP + cat /tmp/tmp.hkmvt8obt2 + rm /tmp/tmp.2DoFfom3wP /tmp/tmp.hkmvt8obt2 + return 0 2026-02-03T15:45:08.500Z INFO setup Manager starting up {"gitCommit": "1e37b17888853ae8e125b9de8658b67d796c4f87", "gitBranch": "PR-2226-1e37b178", "buildTime": "", "goVersion": "go1.25.6", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-sharded-7357 + local namespace=demand-backup-sharded-7357 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-sharded-7357' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-sharded-7357 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-sharded-7357 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.HqwgBu6rNn ++ mktemp + local LAST_OUT=/tmp/tmp.PHIE93Oiyh ++ mktemp + local LAST_ERR=/tmp/tmp.zX81zDYpic + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.onwn9946kU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-sharded-7357 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HqwgBu6rNn + cat /tmp/tmp.zX81zDYpic + rm /tmp/tmp.HqwgBu6rNn /tmp/tmp.zX81zDYpic + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PHIE93Oiyh + cat /tmp/tmp.onwn9946kU + rm /tmp/tmp.PHIE93Oiyh /tmp/tmp.onwn9946kU + return 0 + kubectl_bin wait --for=delete namespace demand-backup-sharded-7357 ++ mktemp + local LAST_OUT=/tmp/tmp.ZY0PqxHUch ++ mktemp + local LAST_ERR=/tmp/tmp.OsSvyNbAFS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-sharded-7357 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZY0PqxHUch + cat /tmp/tmp.OsSvyNbAFS + rm /tmp/tmp.ZY0PqxHUch /tmp/tmp.OsSvyNbAFS + return 0 + desc 'create namespace demand-backup-sharded-7357' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-sharded-7357 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-sharded-7357 ++ mktemp + local LAST_OUT=/tmp/tmp.md0nzPQ0jw ++ mktemp + local LAST_ERR=/tmp/tmp.LuHM7J9LEl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-sharded-7357 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.md0nzPQ0jw namespace/demand-backup-sharded-7357 created + cat /tmp/tmp.LuHM7J9LEl + rm /tmp/tmp.md0nzPQ0jw /tmp/tmp.LuHM7J9LEl + return 0 + set_kube_ctx demand-backup-sharded-7357 + local namespace=demand-backup-sharded-7357 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.PMFhfhvEhX +++ mktemp ++ local LAST_ERR=/tmp/tmp.RIe9T012zM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PMFhfhvEhX ++ cat /tmp/tmp.RIe9T012zM ++ rm /tmp/tmp.PMFhfhvEhX /tmp/tmp.RIe9T012zM ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4 --namespace=demand-backup-sharded-7357 ++ mktemp + local LAST_OUT=/tmp/tmp.PRveR0v6fw ++ mktemp + local LAST_ERR=/tmp/tmp.23qkAQE8df + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4 --namespace=demand-backup-sharded-7357 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PRveR0v6fw Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2226-1e37b178-2-cluster4" modified. + cat /tmp/tmp.23qkAQE8df + rm /tmp/tmp.PRveR0v6fw /tmp/tmp.23qkAQE8df + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Tue Feb 3 15:45:49 2026 NAMESPACE: demand-backup-sharded-7357 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-sharded-7357.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-7357 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-7357 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-7357 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-7357 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AS8t1wLqa0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nenIIHXpgY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AS8t1wLqa0 ++ cat /tmp/tmp.nenIIHXpgY ++ rm /tmp/tmp.AS8t1wLqa0 /tmp/tmp.nenIIHXpgY ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-kxt2f + wait_pod minio-service-6d5f646cdc-kxt2f + local pod=minio-service-6d5f646cdc-kxt2f + set +o xtrace waiting for pod/minio-service-6d5f646cdc-kxt2f to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-7357.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.mpi2r5JvuB ++ mktemp + local LAST_ERR=/tmp/tmp.Mbh0QL6awi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mpi2r5JvuB make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-sharded-7357 namespace + cat /tmp/tmp.Mbh0QL6awi All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-7357 + rm /tmp/tmp.mpi2r5JvuB /tmp/tmp.Mbh0QL6awi + return 0 + desc 'create first PSMDB cluster with custom port 27019' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster with custom port 27019 ----------------------------------------------------------------------------------- + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.liIrMDqMIX ++ mktemp + local LAST_ERR=/tmp/tmp.qy6TQKs2JZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.liIrMDqMIX secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.qy6TQKs2JZ + rm /tmp/tmp.liIrMDqMIX /tmp/tmp.qy6TQKs2JZ + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.IrbZiB7kwW ++ mktemp + local LAST_ERR=/tmp/tmp.mUJzMi3OBk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IrbZiB7kwW secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.mUJzMi3OBk + rm /tmp/tmp.IrbZiB7kwW /tmp/tmp.mUJzMi3OBk + return 0 + version_gt 1.19 ++ echo '1.32 >= 1.19' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' 0 -ne 1 ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/conf/container-rc.yaml + /usr/sbin/sed s/docker/runc/g + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.x3JzIlP4T7 ++ mktemp + local LAST_ERR=/tmp/tmp.vI4ktY1fit + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x3JzIlP4T7 runtimeclass.node.k8s.io/container-rc unchanged + cat /tmp/tmp.vI4ktY1fit + rm /tmp/tmp.x3JzIlP4T7 /tmp/tmp.vI4ktY1fit + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + kubectl_bin apply -f - + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-sharded-7357/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2226-1e37b178"' + local LAST_OUT=/tmp/tmp.HfpzZdJUWA ++ mktemp + local LAST_ERR=/tmp/tmp.dDb7I5mkg2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HfpzZdJUWA perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.dDb7I5mkg2 + rm /tmp/tmp.HfpzZdJUWA /tmp/tmp.dDb7I5mkg2 + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DGVVTWDaFr +++ mktemp ++ local LAST_ERR=/tmp/tmp.kmaiGJRI1F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DGVVTWDaFr ++ cat /tmp/tmp.kmaiGJRI1F ++ rm /tmp/tmp.DGVVTWDaFr /tmp/tmp.kmaiGJRI1F ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gw7GuiDJDM +++ mktemp ++ local LAST_ERR=/tmp/tmp.UTIFju9nW5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gw7GuiDJDM ++ cat /tmp/tmp.UTIFju9nW5 ++ rm /tmp/tmp.gw7GuiDJDM /tmp/tmp.UTIFju9nW5 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X1rkcuNIu2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.4JBkRVEdiL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.X1rkcuNIu2 ++ cat /tmp/tmp.4JBkRVEdiL ++ rm /tmp/tmp.X1rkcuNIu2 /tmp/tmp.4JBkRVEdiL ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness........................................................ + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DYCZTai9XH +++ mktemp ++ local LAST_ERR=/tmp/tmp.4B3v9GBRaI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DYCZTai9XH ++ cat /tmp/tmp.4B3v9GBRaI ++ rm /tmp/tmp.DYCZTai9XH /tmp/tmp.4B3v9GBRaI ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AaGoSEr6g1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VDto72T5E4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AaGoSEr6g1 ++ cat /tmp/tmp.VDto72T5E4 ++ rm /tmp/tmp.AaGoSEr6g1 /tmp/tmp.VDto72T5E4 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3OMiy2o1BS +++ mktemp ++ local LAST_ERR=/tmp/tmp.5IXTEDVpH8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3OMiy2o1BS ++ cat /tmp/tmp.5IXTEDVpH8 ++ rm /tmp/tmp.3OMiy2o1BS /tmp/tmp.5IXTEDVpH8 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rDGNaMcHOS +++ mktemp ++ local LAST_ERR=/tmp/tmp.1tQ2Y0kcly ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rDGNaMcHOS ++ cat /tmp/tmp.1tQ2Y0kcly ++ rm /tmp/tmp.rDGNaMcHOS /tmp/tmp.1tQ2Y0kcly ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yGvruM6D7Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.LDr8S2xFQ2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yGvruM6D7Z ++ cat /tmp/tmp.LDr8S2xFQ2 ++ rm /tmp/tmp.yGvruM6D7Z /tmp/tmp.LDr8S2xFQ2 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LS9fc1kQjv +++ mktemp ++ local LAST_ERR=/tmp/tmp.dJHpFDu7lI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LS9fc1kQjv ++ cat /tmp/tmp.dJHpFDu7lI ++ rm /tmp/tmp.LS9fc1kQjv /tmp/tmp.dJHpFDu7lI ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + sleep 10 + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.C9DahpckLr ++ mktemp + local LAST_ERR=/tmp/tmp.vqjYrf3qhA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.C9DahpckLr + cat /tmp/tmp.vqjYrf3qhA + rm /tmp/tmp.C9DahpckLr /tmp/tmp.vqjYrf3qhA + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-02-03T15:50:55+0000] compare_kubectl: statefulset/some-name-rs0 OK + compare_kubectl statefulset/some-name-rs1 + local resource=statefulset/some-name-rs1 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs1.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs1 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.yOVlwes1lG ++ mktemp + local LAST_ERR=/tmp/tmp.ARICMNiftU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yOVlwes1lG + cat /tmp/tmp.ARICMNiftU + rm /tmp/tmp.yOVlwes1lG /tmp/tmp.ARICMNiftU + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs1.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs1.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs1.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs1.yml + log 'compare_kubectl: statefulset/some-name-rs1 OK' + set +o xtrace [2026-02-03T15:50:56+0000] compare_kubectl: statefulset/some-name-rs1 OK + compare_kubectl statefulset/some-name-rs2 + local resource=statefulset/some-name-rs2 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs2.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs2 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.FWGyGC8kkP ++ mktemp + local LAST_ERR=/tmp/tmp.QVAoATLm7w + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FWGyGC8kkP + cat /tmp/tmp.QVAoATLm7w + rm /tmp/tmp.FWGyGC8kkP /tmp/tmp.QVAoATLm7w + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs2.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs2.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs2.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-rs2.yml + log 'compare_kubectl: statefulset/some-name-rs2 OK' + set +o xtrace [2026-02-03T15:50:57+0000] compare_kubectl: statefulset/some-name-rs2 OK + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.PMjtf4NvzE ++ mktemp + local LAST_ERR=/tmp/tmp.BMkxJ913k3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PMjtf4NvzE + cat /tmp/tmp.BMkxJ913k3 + rm /tmp/tmp.PMjtf4NvzE /tmp/tmp.BMkxJ913k3 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-cfg.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-cfg.yml + log 'compare_kubectl: statefulset/some-name-cfg OK' + set +o xtrace [2026-02-03T15:50:59+0000] compare_kubectl: statefulset/some-name-cfg OK + compare_kubectl statefulset/some-name-mongos '' + local resource=statefulset/some-name-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-mongos + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.D5qcBwFyUO ++ mktemp + local LAST_ERR=/tmp/tmp.8Q9rcjfDOK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D5qcBwFyUO + cat /tmp/tmp.8Q9rcjfDOK + rm /tmp/tmp.D5qcBwFyUO /tmp/tmp.8Q9rcjfDOK + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + log 'compare_kubectl: statefulset/some-name-mongos OK' + set +o xtrace [2026-02-03T15:50:59+0000] compare_kubectl: statefulset/some-name-mongos OK + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.31MRVuEthU ++ mktemp + local LAST_ERR=/tmp/tmp.pznOPluJHv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.31MRVuEthU secret/some-name-mongos created + cat /tmp/tmp.pznOPluJHv + rm /tmp/tmp.31MRVuEthU /tmp/tmp.pznOPluJHv + return 0 + sleep 10 + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q1Ld985FM0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gFl9lpgdtu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q1Ld985FM0 ++ cat /tmp/tmp.gFl9lpgdtu ++ rm /tmp/tmp.Q1Ld985FM0 /tmp/tmp.gFl9lpgdtu ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1YfaQkRIGG +++ mktemp ++ local LAST_ERR=/tmp/tmp.J65XBp9EuM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1YfaQkRIGG ++ cat /tmp/tmp.J65XBp9EuM ++ rm /tmp/tmp.1YfaQkRIGG /tmp/tmp.J65XBp9EuM ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OmLHVo4lla +++ mktemp ++ local LAST_ERR=/tmp/tmp.RcJfeaxvEM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OmLHVo4lla ++ cat /tmp/tmp.RcJfeaxvEM ++ rm /tmp/tmp.OmLHVo4lla /tmp/tmp.RcJfeaxvEM ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................................................................................. + compare_kubectl statefulset/some-name-mongos -secret + local resource=statefulset/some-name-mongos + local postfix=-secret + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml + local new_result=/tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-mongos + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-7357", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.I7TF9vfHTc ++ mktemp + local LAST_ERR=/tmp/tmp.vybbiXYqV3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I7TF9vfHTc + cat /tmp/tmp.vybbiXYqV3 + rm /tmp/tmp.I7TF9vfHTc /tmp/tmp.vybbiXYqV3 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + version_gt 1.22 ++ echo '1.32 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.35T1HHv8Nw/statefulset_some-name-mongos.yml + log 'compare_kubectl: statefulset/some-name-mongos OK' + set +o xtrace [2026-02-03T15:55:06+0000] compare_kubectl: statefulset/some-name-mongos OK + desc 'checking hookscripts' + set +o xtrace ----------------------------------------------------------------------------------- checking hookscripts ----------------------------------------------------------------------------------- + kubectl_bin logs some-name-rs0-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.pjQajWyd3z ++ mktemp + local LAST_ERR=/tmp/tmp.VrvRYTq1bC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs0-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pjQajWyd3z + cat /tmp/tmp.VrvRYTq1bC + rm /tmp/tmp.pjQajWyd3z /tmp/tmp.VrvRYTq1bC + return 0 + kubectl_bin logs some-name-rs0-0 + grep -q 'RS0 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.6Tjt41EDPU ++ mktemp + local LAST_ERR=/tmp/tmp.pzEL3eR2Mr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs0-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6Tjt41EDPU cat: write error: Broken pipe + cat /tmp/tmp.pzEL3eR2Mr Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) + rm /tmp/tmp.6Tjt41EDPU /tmp/tmp.pzEL3eR2Mr + return 0 + kubectl_bin logs some-name-rs1-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.bLFXmsGQKk ++ mktemp + local LAST_ERR=/tmp/tmp.dmV4Fa7SRN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs1-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bLFXmsGQKk + cat /tmp/tmp.dmV4Fa7SRN + rm /tmp/tmp.bLFXmsGQKk /tmp/tmp.dmV4Fa7SRN + return 0 + kubectl_bin logs some-name-rs1-0 + grep -q 'RS1 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.eVvAAMDVMk ++ mktemp + local LAST_ERR=/tmp/tmp.fuMx45j2I4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs1-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eVvAAMDVMk cat: write error: Broken pipe + cat /tmp/tmp.fuMx45j2I4 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) + rm /tmp/tmp.eVvAAMDVMk /tmp/tmp.fuMx45j2I4 + return 0 + kubectl_bin logs some-name-rs2-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.ziKstt9lLI ++ mktemp + local LAST_ERR=/tmp/tmp.SqXHadoudR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs2-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ziKstt9lLI + cat /tmp/tmp.SqXHadoudR + rm /tmp/tmp.ziKstt9lLI /tmp/tmp.SqXHadoudR + return 0 + kubectl_bin logs some-name-rs2-0 + grep -q 'RS2 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.HOSo4uEHcb ++ mktemp + local LAST_ERR=/tmp/tmp.FAFTH90x1l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs2-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HOSo4uEHcb cat: write error: Broken pipe + cat /tmp/tmp.FAFTH90x1l Defaulted container "mongod" out of: mongod, rs-sidecar-1, backup-agent, mongo-init (init) + rm /tmp/tmp.HOSo4uEHcb /tmp/tmp.FAFTH90x1l + return 0 + kubectl_bin logs some-name-cfg-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.6cd9xTbtBx ++ mktemp + local LAST_ERR=/tmp/tmp.csPZz0bMHC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-cfg-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6cd9xTbtBx + cat /tmp/tmp.csPZz0bMHC + rm /tmp/tmp.6cd9xTbtBx /tmp/tmp.csPZz0bMHC + return 0 + kubectl_bin logs some-name-cfg-0 + grep -q 'CFG Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.ObfHOQAamh ++ mktemp + local LAST_ERR=/tmp/tmp.vQOC0lyrtS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-cfg-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ObfHOQAamh cat: write error: Broken pipe + cat /tmp/tmp.vQOC0lyrtS Defaulted container "mongod" out of: mongod, cfg-sidecar-1, backup-agent, mongo-init (init) + rm /tmp/tmp.ObfHOQAamh /tmp/tmp.vQOC0lyrtS + return 0 + kubectl_bin logs some-name-mongos-0 + grep -q 'Mongos Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.YZgo24hIAw ++ mktemp + local LAST_ERR=/tmp/tmp.MveH2f0vVI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-mongos-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YZgo24hIAw cat: write error: Broken pipe + cat /tmp/tmp.MveH2f0vVI Defaulted container "mongos" out of: mongos, mongos-sidecar-1, mongo-init (init) + rm /tmp/tmp.YZgo24hIAw /tmp/tmp.MveH2f0vVI + return 0 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9xkJPZayr0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yxwH3Afva9 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9xkJPZayr0 ++ cat /tmp/tmp.yxwH3Afva9 ++ rm /tmp/tmp.9xkJPZayr0 /tmp/tmp.yxwH3Afva9 ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.DDk0bSZ49D ++ mktemp + local LAST_ERR=/tmp/tmp.1PCZsZgGmq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DDk0bSZ49D Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8135dea3-ae3f-4e1a-baf8-679fe3d47bd4") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" }, { "db" : "myApp1", "role" : "readWrite" }, { "db" : "myApp2", "role" : "readWrite" } ] } bye + cat /tmp/tmp.1PCZsZgGmq + rm /tmp/tmp.DDk0bSZ49D /tmp/tmp.1PCZsZgGmq + return 0 + run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=sh.enableSharding("myApp","rs0")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RHIKv9HJT5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.oaML7EWg1Q ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RHIKv9HJT5 ++ cat /tmp/tmp.oaML7EWg1Q ++ rm /tmp/tmp.RHIKv9HJT5 /tmp/tmp.oaML7EWg1Q ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.i7Pk8M8wcx ++ mktemp + local LAST_ERR=/tmp/tmp.ONXy4Fqwt4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.i7Pk8M8wcx Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("28683700-6710-43c8-a3fb-83730adeebdb") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1770134127, 11), "signature" : { "hash" : BinData(0,"Y5j9TQYriAf+q1ZUGvfqDPb3MdM="), "keyId" : NumberLong("7602666235083358232") } }, "operationTime" : Timestamp(1770134127, 8) } bye + cat /tmp/tmp.ONXy4Fqwt4 + rm /tmp/tmp.i7Pk8M8wcx /tmp/tmp.ONXy4Fqwt4 + return 0 + run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=sh.enableSharding("myApp1","rs1")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hGQqEKKNDG +++ mktemp ++ local LAST_ERR=/tmp/tmp.yZcnh0JepD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hGQqEKKNDG ++ cat /tmp/tmp.yZcnh0JepD ++ rm /tmp/tmp.hGQqEKKNDG /tmp/tmp.yZcnh0JepD ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Fhcs7KrkT5 ++ mktemp + local LAST_ERR=/tmp/tmp.JOK77gROXY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Fhcs7KrkT5 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("61c7812d-0f2b-41c1-92a6-4635ba9a8ca9") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1770134130, 11), "signature" : { "hash" : BinData(0,"xZLOPwKT5MlQIrsDNJZdh4be1CM="), "keyId" : NumberLong("7602666235083358232") } }, "operationTime" : Timestamp(1770134130, 8) } bye + cat /tmp/tmp.JOK77gROXY + rm /tmp/tmp.Fhcs7KrkT5 /tmp/tmp.JOK77gROXY + return 0 + run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=sh.enableSharding("myApp2","rs2")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C2HK3oRSy6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EOOEgdwOIG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C2HK3oRSy6 ++ cat /tmp/tmp.EOOEgdwOIG ++ rm /tmp/tmp.C2HK3oRSy6 /tmp/tmp.EOOEgdwOIG ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.MCGmBf62lE ++ mktemp + local LAST_ERR=/tmp/tmp.Lqk2xq9cOA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MCGmBf62lE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("7f8dfdf4-124e-47ac-91f5-a4bcc73adfa6") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1770134133, 10), "signature" : { "hash" : BinData(0,"HnKgK8fmaj3h7MeUaR7G9uGYs44="), "keyId" : NumberLong("7602666235083358232") } }, "operationTime" : Timestamp(1770134133, 7) } bye + cat /tmp/tmp.Lqk2xq9cOA + rm /tmp/tmp.MCGmBf62lE /tmp/tmp.Lqk2xq9cOA + return 0 + insert_data_mongos 100500 myApp '' 27019 + local data=100500 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XpVHfg8yap +++ mktemp ++ local LAST_ERR=/tmp/tmp.bsVKb80t4v ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XpVHfg8yap ++ cat /tmp/tmp.bsVKb80t4v ++ rm /tmp/tmp.XpVHfg8yap /tmp/tmp.bsVKb80t4v ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.hER8k4lcY8 ++ mktemp + local LAST_ERR=/tmp/tmp.4pvLGqiFEJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hER8k4lcY8 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3c8ba7d4-54a0-45ec-989c-389d8a4ca34b") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.4pvLGqiFEJ + rm /tmp/tmp.hER8k4lcY8 /tmp/tmp.4pvLGqiFEJ + return 0 + insert_data_mongos 100500 myApp1 '' 27019 + local data=100500 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kVDUI8iOZy +++ mktemp ++ local LAST_ERR=/tmp/tmp.3zjiIkuSiZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kVDUI8iOZy ++ cat /tmp/tmp.3zjiIkuSiZ ++ rm /tmp/tmp.kVDUI8iOZy /tmp/tmp.3zjiIkuSiZ ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.WNUdoBisYD ++ mktemp + local LAST_ERR=/tmp/tmp.wPS2NdjPJt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WNUdoBisYD Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8f77ea1e-411f-4dae-92a1-160accb724ba") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.wPS2NdjPJt + rm /tmp/tmp.WNUdoBisYD /tmp/tmp.wPS2NdjPJt + return 0 + insert_data_mongos 100500 myApp2 '' 27019 + local data=100500 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-7357 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eaxydPqXx0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.syR0FfPj1L ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eaxydPqXx0 ++ cat /tmp/tmp.syR0FfPj1L ++ rm /tmp/tmp.eaxydPqXx0 /tmp/tmp.syR0FfPj1L ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.hUX5ip65aS ++ mktemp + local LAST_ERR=/tmp/tmp.h5BOgXOJUp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hUX5ip65aS Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("94bd8975-f030-4054-a7c1-41a8e4d20394") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.h5BOgXOJUp + rm /tmp/tmp.hUX5ip65aS /tmp/tmp.h5BOgXOJUp + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-7357 '' '' '' '' 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-7357 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27019 + local tls=false + [[ false == \t\r\u\e ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-02-03T15:55:41+0000] running db.test.command() in myApp + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-7357 mongodb '' '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-7357 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v45YaHcVTp +++ mktemp ++ local LAST_ERR=/tmp/tmp.yhdIIZ9Gll ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v45YaHcVTp ++ cat /tmp/tmp.yhdIIZ9Gll ++ rm /tmp/tmp.v45YaHcVTp /tmp/tmp.yhdIIZ9Gll ++ return 0 + local client_container=psmdb-client-86cb5d8484-nxfjg + kubectl_bin exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ipqkpKOkl3 ++ mktemp + local LAST_ERR=/tmp/tmp.s5S7FqGFl3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-nxfjg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-7357.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ipqkpKOkl3 + cat /tmp/tmp.s5S7FqGFl3 + rm /tmp/tmp.ipqkpKOkl3 /tmp/tmp.s5S7FqGFl3 + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.35T1HHv8Nw/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-02-03T15:49:47.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-02-03T15:49:50.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-02-03T15:50:01.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2026-02-03T15:49:47.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2026-02-03T15:49:50.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2026-02-03T15:50:01.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2026-02-03T15:49:47.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2026-02-03T15:49:52.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2026-02-03T15:50:02.000+0000 I listening for the commands + backup_name_aws=backup-aws-s3 + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + backup_name_azure=backup-azure-blob + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-02-03T15:55:56+0000] running backup backup-minio + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.wVEOCqzLFg ++ mktemp + local LAST_ERR=/tmp/tmp.Fq2qw0iMlI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wVEOCqzLFg perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.Fq2qw0iMlI + rm /tmp/tmp.wVEOCqzLFg /tmp/tmp.Fq2qw0iMlI + return 0 + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-02-03T15:55:58+0000] running backup backup-aws-s3 + yq eval '.metadata.name = "backup-aws-s3" | .spec.storageName = "aws-s3" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.IXb6b78dhV ++ mktemp + local LAST_ERR=/tmp/tmp.7El4yTEVp8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IXb6b78dhV perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.7El4yTEVp8 + rm /tmp/tmp.IXb6b78dhV /tmp/tmp.7El4yTEVp8 + return 0 + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + log 'running backup backup-gcp-cs' + set +o xtrace [2026-02-03T15:56:00+0000] running backup backup-gcp-cs + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Rj82TPOiZp ++ mktemp + local LAST_ERR=/tmp/tmp.fBdD4q5rfG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Rj82TPOiZp perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.fBdD4q5rfG + rm /tmp/tmp.Rj82TPOiZp /tmp/tmp.fBdD4q5rfG + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-02-03T15:56:03+0000] running backup backup-azure-blob + yq eval '.metadata.name = "backup-azure-blob" | .spec.storageName = "azure-blob" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2226/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.u0YKvDSx05 ++ mktemp + local LAST_ERR=/tmp/tmp.mpDrzfhSet + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u0YKvDSx05 perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.mpDrzfhSet + rm /tmp/tmp.u0YKvDSx05 /tmp/tmp.mpDrzfhSet + return 0 + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2026-02-03T16:15:29.846Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "5330634b-5b92-48ee-a80d-f32083891d64", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:29.928Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "5330634b-5b92-48ee-a80d-f32083891d64"} 2026-02-03T16:15:29.928Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "5330634b-5b92-48ee-a80d-f32083891d64", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:29.928Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "5330634b-5b92-48ee-a80d-f32083891d64"} 2026-02-03T16:15:34.673Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8"} 2026-02-03T16:15:34.684Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8", "state": "running"} 2026-02-03T16:15:34.775Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:34.775Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:15:34.859Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8"} 2026-02-03T16:15:34.860Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:15:34.863Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "c4d5b103-20e4-468b-9ca8-67e7d759cdc8"} 2026-02-03T16:15:34.929Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a"} 2026-02-03T16:15:34.942Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a", "state": "waiting"} 2026-02-03T16:15:35.032Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:35.032Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:35.112Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a"} 2026-02-03T16:15:35.112Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:35.113Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3c706719-2e55-4427-a311-fc19497dbe0a"} 2026-02-03T16:15:39.863Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc"} 2026-02-03T16:15:39.877Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc", "state": "running"} 2026-02-03T16:15:39.949Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:39.949Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:15:40.037Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc"} 2026-02-03T16:15:40.038Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:15:40.040Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "fb2af1a0-44f1-4e6e-a489-bb9dc477b1cc"} 2026-02-03T16:15:40.114Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414"} 2026-02-03T16:15:40.125Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414", "state": "waiting"} 2026-02-03T16:15:40.227Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:40.227Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:40.311Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414"} 2026-02-03T16:15:40.311Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:40.312Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "9b115920-8703-46d3-afc7-3274a666c414"} 2026-02-03T16:15:45.041Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c"} 2026-02-03T16:15:45.053Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c", "state": "running"} 2026-02-03T16:15:45.129Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:45.129Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:15:45.215Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c"} 2026-02-03T16:15:45.216Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:15:45.218Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "8d831ae2-0a35-48d2-ad0c-d9c1d4cf929c"} 2026-02-03T16:15:45.313Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca"} 2026-02-03T16:15:45.324Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca", "state": "waiting"} 2026-02-03T16:15:45.446Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:45.446Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:45.574Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca"} 2026-02-03T16:15:45.575Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:45.576Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0431c35b-381d-4dfe-abb3-63a97d94dcca"} 2026-02-03T16:15:50.218Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e"} 2026-02-03T16:15:50.228Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e", "state": "running"} 2026-02-03T16:15:50.309Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:50.309Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:15:50.409Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e"} 2026-02-03T16:15:50.410Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:15:50.419Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "eb7b63a4-4b5c-4740-bdd4-5c3a7118669e"} 2026-02-03T16:15:50.577Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5"} 2026-02-03T16:15:50.589Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5", "state": "waiting"} 2026-02-03T16:15:50.670Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:50.670Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:50.758Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5"} 2026-02-03T16:15:50.759Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:50.759Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "7b1889ad-5a1c-4232-a088-495ffdd058e5"} 2026-02-03T16:15:55.420Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7"} 2026-02-03T16:15:55.440Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7", "state": "running"} 2026-02-03T16:15:55.540Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:55.540Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:15:55.623Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7"} 2026-02-03T16:15:55.626Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:15:55.630Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "e2b6d41d-cea4-4cf6-8f43-3e594ff39af7"} 2026-02-03T16:15:55.761Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c"} 2026-02-03T16:15:55.777Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c", "state": "waiting"} 2026-02-03T16:15:55.863Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:15:55.863Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:15:55.936Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c"} 2026-02-03T16:15:55.936Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:15:55.938Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "cdd6e398-a12a-4c27-ae60-494cbf1c605c"} 2026-02-03T16:16:00.631Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe"} 2026-02-03T16:16:00.643Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe", "state": "running"} 2026-02-03T16:16:00.752Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:00.753Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:00.869Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe"} 2026-02-03T16:16:00.870Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:00.872Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "01da34c2-d0c0-43f0-9e53-0d1b20e0c0fe"} 2026-02-03T16:16:00.939Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d"} 2026-02-03T16:16:00.948Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d", "state": "waiting"} 2026-02-03T16:16:01.040Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:01.040Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:01.120Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d"} 2026-02-03T16:16:01.120Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:01.121Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e233bd12-bff1-44ed-ad26-7b665b4a9a0d"} 2026-02-03T16:16:05.872Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db"} 2026-02-03T16:16:05.885Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db", "state": "running"} 2026-02-03T16:16:05.969Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:05.970Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:06.060Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db"} 2026-02-03T16:16:06.061Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:06.063Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "d99a41de-87ab-4d7f-9114-204d67ced7db"} 2026-02-03T16:16:06.122Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4"} 2026-02-03T16:16:06.134Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4", "state": "waiting"} 2026-02-03T16:16:06.205Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:06.205Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:06.290Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4"} 2026-02-03T16:16:06.290Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:06.291Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e41815c5-8a03-4a2b-ab6b-58afc57c3fa4"} 2026-02-03T16:16:11.064Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94"} 2026-02-03T16:16:11.076Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94", "state": "running"} 2026-02-03T16:16:11.149Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:11.149Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:11.227Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94"} 2026-02-03T16:16:11.228Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:11.230Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "b3917f1e-156c-4a46-840b-25f59ffdae94"} 2026-02-03T16:16:11.292Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af"} 2026-02-03T16:16:11.302Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af", "state": "waiting"} 2026-02-03T16:16:11.377Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:11.377Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:11.467Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af"} 2026-02-03T16:16:11.467Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:11.468Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "3aa32a71-063c-41da-a87c-65a0df13f1af"} 2026-02-03T16:16:16.231Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138"} 2026-02-03T16:16:16.242Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138", "state": "running"} 2026-02-03T16:16:16.334Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:16.334Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:16.439Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138"} 2026-02-03T16:16:16.462Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:16.466Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "0937aa87-8071-4c3a-97d2-144066bb4138"} 2026-02-03T16:16:16.468Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16"} 2026-02-03T16:16:16.481Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16", "state": "waiting"} 2026-02-03T16:16:16.608Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:16.608Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:16.709Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16"} 2026-02-03T16:16:16.709Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:16.711Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e6f7d0f4-3a77-4d2c-b4fd-23602bcb6a16"} 2026-02-03T16:16:21.467Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a"} 2026-02-03T16:16:21.481Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a", "state": "running"} 2026-02-03T16:16:21.597Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:21.597Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:21.671Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a"} 2026-02-03T16:16:21.672Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:21.674Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "871c6a63-1f8a-4eb2-a3d1-16bad6bddb8a"} 2026-02-03T16:16:21.712Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57"} 2026-02-03T16:16:21.724Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57", "state": "waiting"} 2026-02-03T16:16:21.809Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:21.809Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:21.886Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57"} 2026-02-03T16:16:21.886Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:21.888Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "f280f300-1ac2-4dfd-97b2-fac09fae5a57"} 2026-02-03T16:16:26.674Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4"} 2026-02-03T16:16:26.690Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4", "state": "running"} 2026-02-03T16:16:26.806Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:26.806Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:26.946Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4"} 2026-02-03T16:16:26.947Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:26.949Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "1d6c86cc-ad8a-4462-a3e1-23616b8a52d4"} 2026-02-03T16:16:26.949Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83"} 2026-02-03T16:16:26.966Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83", "state": "waiting"} 2026-02-03T16:16:27.055Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:27.055Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:27.127Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83"} 2026-02-03T16:16:27.127Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:27.129Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e2b5a773-afe6-4476-8d01-8144bc8eae83"} 2026-02-03T16:16:31.950Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46"} 2026-02-03T16:16:31.962Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46", "state": "running"} 2026-02-03T16:16:32.054Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:32.054Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:32.144Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46"} 2026-02-03T16:16:32.145Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:32.147Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "081c82d8-e64d-440a-85c2-2c840cdccd46"} 2026-02-03T16:16:32.147Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7"} 2026-02-03T16:16:32.158Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7", "state": "waiting"} 2026-02-03T16:16:32.275Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:32.275Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:32.356Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7"} 2026-02-03T16:16:32.356Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:32.358Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "0197bc14-f0be-4514-a8c8-03b1852d4dc7"} 2026-02-03T16:16:37.148Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5"} 2026-02-03T16:16:37.160Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5", "state": "running"} 2026-02-03T16:16:37.234Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:37.234Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:37.311Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5"} 2026-02-03T16:16:37.312Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:37.314Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "a5d5f413-d476-4a8d-8bf1-9253241c5eb5"} 2026-02-03T16:16:37.358Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720"} 2026-02-03T16:16:37.369Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720", "state": "waiting"} 2026-02-03T16:16:37.456Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:37.456Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:37.555Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720"} 2026-02-03T16:16:37.555Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:37.556Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "a2ca0663-58d6-44f1-8c70-77ac77b88720"} 2026-02-03T16:16:42.314Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff"} 2026-02-03T16:16:42.329Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff", "state": "running"} 2026-02-03T16:16:42.471Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:42.471Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2026-02-03T16:16:42.579Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff"} 2026-02-03T16:16:42.581Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff", "backup": "backup-aws-s3", "pbmName": "2026-02-03T15:57:04Z", "meta": {"type":"logical","opid":"69821ad04c69fb06fb8a22d4","name":"2026-02-03T15:57:04Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2026-02-03T15:57:04Z/rs1/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs1/oplog","start_ts":1770134224,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134282,"first_write_ts":{"T":1770134211,"I":5},"last_write_ts":{"T":1770134228,"I":9},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134224,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134282,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs2","backup_name":"2026-02-03T15:57:04Z/rs2/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs2/oplog","start_ts":1770134225,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1770134281,"first_write_ts":{"T":1770134211,"I":10},"last_write_ts":{"T":1770134229,"I":9},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","error":"waiting for done: backup stuck, last beat ts: 1770134250","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"},{"timestamp":1770134281,"status":"error","error":"waiting for done: backup stuck, last beat ts: 1770134250"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"rs0","backup_name":"2026-02-03T15:57:04Z/rs0/metadata.json","oplog_name":"2026-02-03T15:57:04Z/rs0/oplog","start_ts":1770134225,"status":"done","size":0,"size_uncompressed":0,"last_transition_ts":1770134241,"first_write_ts":{"T":1770134223,"I":733},"last_write_ts":{"T":1770134229,"I":29},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"},{"timestamp":1770134241,"status":"done"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"},{"name":"cfg","backup_name":"2026-02-03T15:57:04Z/cfg/metadata.json","oplog_name":"2026-02-03T15:57:04Z/cfg/oplog","start_ts":1770134225,"status":"dumpDone","size":0,"size_uncompressed":0,"iscs":true,"last_transition_ts":1770134229,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1770134230,"I":12},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","conditions":[{"timestamp":1770134225,"status":"running"},{"timestamp":1770134229,"status":"dumpDone"}],"pbm_version":"2.12.0","mongo_version":"8.0.17-6"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":36771,"size_uncompressed":0,"mongodb_version":"8.0.17-6","fcv":"8.0","start_ts":1770134224,"last_transition_ts":1770134230,"first_write_ts":{"T":1770134225,"I":12},"last_write_ts":{"T":1,"I":1},"hb":{"T":1770134250,"I":2},"status":"dumpDone","conditions":[{"timestamp":1770134224,"status":"starting"},{"timestamp":1770134228,"status":"running"},{"timestamp":1770134230,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016","some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016"}],"pbm_version":"2.12.0","balancer":"full"}} 2026-02-03T16:16:42.583Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-aws-s3", "reconcileID": "2202dd73-4abd-443b-a21a-71d13d9157ff"} 2026-02-03T16:16:42.583Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4"} 2026-02-03T16:16:42.614Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4", "state": "waiting"} 2026-02-03T16:16:42.723Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4", "cluster": "some-name", "namespace": "demand-backup-sharded-7357"} 2026-02-03T16:16:42.723Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2026-02-03T16:16:42.811Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4"} 2026-02-03T16:16:42.811Z INFO Another backup is holding the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4", "holder": "backup-aws-s3-390773c3-6c51-4ed9-be73-5308147974f7"} 2026-02-03T16:16:42.812Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-7357"}, "namespace": "demand-backup-sharded-7357", "name": "backup-gcp-cs", "reconcileID": "e684d310-c964-43c6-bf4c-22c01fb8b4f4"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-aws-s3 some-name aws-s3 s3://operator-testing/psmdb-demand-backup-sharded/2026-02-03T15:57:04Z logical running 20m backup-azure-blob some-name azure-blob azure://operator-testing/psmdb-demand-backup-sharded/2026-02-03T15:56:32Z logical 161.96KB ready 19m 20m backup-gcp-cs some-name gcp-cs waiting 20m backup-minio some-name minio s3://operator-testing/2026-02-03T15:55:58Z logical 157.95KB ready 20m 20m Name: backup-aws-s3 Namespace: demand-backup-sharded-7357 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-02-03T15:56:00Z Finalizers: percona.com/delete-backup internal.percona.com/release-lock Generation: 1 Resource Version: 1770134229592639001 UID: 390773c3-6c51-4ed9-be73-5308147974f7 Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: aws-s3 Type: logical Status: Destination: s3://operator-testing/psmdb-demand-backup-sharded/2026-02-03T15:57:04Z Last Transition: 2026-02-03T15:57:08Z Pbm Name: 2026-02-03T15:57:04Z Pbm Pod: some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016 Pbm Pods: Cfg: some-name-cfg-1.some-name-cfg.demand-backup-sharded-7357.svc.cluster.local:27018 rs0: some-name-rs0-2.some-name-rs0.demand-backup-sharded-7357.svc.cluster.local:27016 rs1: some-name-rs1-2.some-name-rs1.demand-backup-sharded-7357.svc.cluster.local:27016 rs2: some-name-rs2-2.some-name-rs2.demand-backup-sharded-7357.svc.cluster.local:27016 Replset Names: cfg rs0 rs1 rs2 s3: Bucket: operator-testing Credentials Secret: aws-s3-secret Prefix: psmdb-demand-backup-sharded Region: us-east-1 Server Side Encryption: Start: 2026-02-03T15:57:04Z State: running Storage Name: aws-s3 Type: logical Events: Backup object psmdb-backup/backup-aws-s3 is in running state.