Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/logs/demand-backup-incremental-sharded.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-incremental-sharded-13280
+ local ns=demand-backup-incremental-sharded-13280
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.M4HyChhJ9S
++ mktemp
+ local LAST_ERR=/tmp/tmp.FaWzt349Ka
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M4HyChhJ9S
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.FaWzt349Ka
+ rm /tmp/tmp.M4HyChhJ9S /tmp/tmp.FaWzt349Ka
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.D25YsXsAxF
++ mktemp
+ local LAST_ERR=/tmp/tmp.gyhPAaIDt7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D25YsXsAxF
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.gyhPAaIDt7
+ rm /tmp/tmp.D25YsXsAxF /tmp/tmp.gyhPAaIDt7
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rsdsm0IR52
++ mktemp
+ local LAST_ERR=/tmp/tmp.f53DuDRjit
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Rsdsm0IR52
+ cat /tmp/tmp.f53DuDRjit
+ rm /tmp/tmp.Rsdsm0IR52 /tmp/tmp.f53DuDRjit
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.B3VL9trDIP
++ mktemp
+ local LAST_ERR=/tmp/tmp.6mdTFlVRKC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B3VL9trDIP
+ cat /tmp/tmp.6mdTFlVRKC
+ rm /tmp/tmp.B3VL9trDIP /tmp/tmp.6mdTFlVRKC
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.mkBImSyIv4
++ mktemp
+ local LAST_ERR=/tmp/tmp.kw9585m1KW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mkBImSyIv4
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.kw9585m1KW
+ rm /tmp/tmp.mkBImSyIv4 /tmp/tmp.kw9585m1KW
+ return 0
+ check_crd_for_deletion PR-1904-a4e779d3
+ local git_tag=PR-1904-a4e779d3
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1904-a4e779d3/deploy/crd.yaml
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/bin/sed s/---//g
++ yq eval .metadata.name
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jSJAyy4asL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AkZMT66pxC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jSJAyy4asL
++ cat /tmp/tmp.AkZMT66pxC
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jSJAyy4asL
++ cat /tmp/tmp.AkZMT66pxC
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.jSJAyy4asL
++ cat /tmp/tmp.AkZMT66pxC
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.jSJAyy4asL
++ cat /tmp/tmp.AkZMT66pxC
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.jSJAyy4asL /tmp/tmp.AkZMT66pxC
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.fG2G2sMX3Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.1ZsekmFgB4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
++ mktemp
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ local LAST_OUT=/tmp/tmp.93U4eU5cpa
++ mktemp
+ local LAST_ERR=/tmp/tmp.uMIrJTN15G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ awk '{print$1}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.93U4eU5cpa
+ cat /tmp/tmp.uMIrJTN15G
+ rm /tmp/tmp.93U4eU5cpa /tmp/tmp.uMIrJTN15G
+ return 0
namespace "demand-backup-incremental-sharded-18554" deleted
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fG2G2sMX3Q
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.1ZsekmFgB4
+ rm /tmp/tmp.fG2G2sMX3Q /tmp/tmp.1ZsekmFgB4
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.l0FrDecz97
++ mktemp
+ local LAST_ERR=/tmp/tmp.54SNCUeFEn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l0FrDecz97
+ cat /tmp/tmp.54SNCUeFEn
+ rm /tmp/tmp.l0FrDecz97 /tmp/tmp.54SNCUeFEn
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.RdQqYs5qbj
++ mktemp
+ local LAST_ERR=/tmp/tmp.iqvx6hqPZK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RdQqYs5qbj
namespace/psmdb-operator created
+ cat /tmp/tmp.iqvx6hqPZK
+ rm /tmp/tmp.RdQqYs5qbj /tmp/tmp.iqvx6hqPZK
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YD40tVWACa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XiVQpzbGb2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YD40tVWACa
++ cat /tmp/tmp.XiVQpzbGb2
++ rm /tmp/tmp.YD40tVWACa /tmp/tmp.XiVQpzbGb2
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jmmuQ6wOvl
++ mktemp
+ local LAST_ERR=/tmp/tmp.6U4v7z7ng2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jmmuQ6wOvl
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1" modified.
+ cat /tmp/tmp.6U4v7z7ng2
+ rm /tmp/tmp.jmmuQ6wOvl /tmp/tmp.6U4v7z7ng2
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UsfnxTrFqF
++ mktemp
+ local LAST_ERR=/tmp/tmp.UPwQMQ3txj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UsfnxTrFqF
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.UPwQMQ3txj
+ rm /tmp/tmp.UsfnxTrFqF /tmp/tmp.UPwQMQ3txj
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Usv7Wveust
++ mktemp
+ local LAST_ERR=/tmp/tmp.De8Vxdbwr6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Usv7Wveust
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.De8Vxdbwr6
+ rm /tmp/tmp.Usv7Wveust /tmp/tmp.De8Vxdbwr6
+ return 0
+ kubectl_bin apply -f -
++ mktemp
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1904-a4e779d3") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-operator.yaml
+ local LAST_OUT=/tmp/tmp.t1lc9QtL8U
++ mktemp
+ local LAST_ERR=/tmp/tmp.XBqNdpJWrK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t1lc9QtL8U
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.XBqNdpJWrK
+ rm /tmp/tmp.t1lc9QtL8U /tmp/tmp.XBqNdpJWrK
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ykrtgevPth
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F9HVjz2pX1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ykrtgevPth
++ cat /tmp/tmp.F9HVjz2pX1
++ rm /tmp/tmp.ykrtgevPth /tmp/tmp.F9HVjz2pX1
++ return 0
+ wait_pod percona-server-mongodb-operator-85c5b67d9d-8sqkr
+ local pod=percona-server-mongodb-operator-85c5b67d9d-8sqkr
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-85c5b67d9d-8sqkr to be ready..OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xSskj4M6Ab
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Egvco3FqK3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xSskj4M6Ab
++ cat /tmp/tmp.Egvco3FqK3
++ rm /tmp/tmp.xSskj4M6Ab /tmp/tmp.Egvco3FqK3
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-85c5b67d9d-8sqkr
++ mktemp
+ local LAST_OUT=/tmp/tmp.NQsqB08cnl
++ mktemp
+ local LAST_ERR=/tmp/tmp.lOcRIqIAiS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-85c5b67d9d-8sqkr
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NQsqB08cnl
+ cat /tmp/tmp.lOcRIqIAiS
+ rm /tmp/tmp.NQsqB08cnl /tmp/tmp.lOcRIqIAiS
+ return 0
2025-05-14T16:07:48.609Z	INFO	setup	Manager starting up	{"gitCommit": "a4e779d35a323625e9652ede4a65877e643ad719", "gitBranch": "PR-1904-a4e779d3", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-incremental-sharded-13280
+ local namespace=demand-backup-incremental-sharded-13280
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-incremental-sharded-13280'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-incremental-sharded-13280
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-incremental-sharded-13280 --ignore-not-found
+ xargs kubectl delete ns
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.8j5ICQsZHy
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.aZGCZWOJ87
+ local exit_status=0
+ local timeout=4
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.Gd3GC24Way
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.vPkrcaDOIp
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-incremental-sharded-13280 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8j5ICQsZHy
+ cat /tmp/tmp.aZGCZWOJ87
+ rm /tmp/tmp.8j5ICQsZHy /tmp/tmp.aZGCZWOJ87
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gd3GC24Way
+ cat /tmp/tmp.vPkrcaDOIp
+ rm /tmp/tmp.Gd3GC24Way /tmp/tmp.vPkrcaDOIp
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-incremental-sharded-13280
++ mktemp
+ local LAST_OUT=/tmp/tmp.XDFU7cA36o
++ mktemp
+ local LAST_ERR=/tmp/tmp.QkcPz33sEU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-incremental-sharded-13280
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XDFU7cA36o
+ cat /tmp/tmp.QkcPz33sEU
+ rm /tmp/tmp.XDFU7cA36o /tmp/tmp.QkcPz33sEU
+ return 0
+ desc 'create namespace demand-backup-incremental-sharded-13280'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-incremental-sharded-13280
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-incremental-sharded-13280
++ mktemp
+ local LAST_OUT=/tmp/tmp.E0O4hsoYD3
++ mktemp
+ local LAST_ERR=/tmp/tmp.qnqMYaTqyy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-incremental-sharded-13280
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E0O4hsoYD3
namespace/demand-backup-incremental-sharded-13280 created
+ cat /tmp/tmp.qnqMYaTqyy
+ rm /tmp/tmp.E0O4hsoYD3 /tmp/tmp.qnqMYaTqyy
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FjAKGmEDx4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ADVTzg8qYh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FjAKGmEDx4
++ cat /tmp/tmp.ADVTzg8qYh
++ rm /tmp/tmp.FjAKGmEDx4 /tmp/tmp.ADVTzg8qYh
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1 --namespace=demand-backup-incremental-sharded-13280
++ mktemp
+ local LAST_OUT=/tmp/tmp.NHhg1tVckF
++ mktemp
+ local LAST_ERR=/tmp/tmp.QXj4XDxRwu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1 --namespace=demand-backup-incremental-sharded-13280
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NHhg1tVckF
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-a4e779d3-11-cluster1" modified.
+ cat /tmp/tmp.QXj4XDxRwu
+ rm /tmp/tmp.NHhg1tVckF /tmp/tmp.QXj4XDxRwu
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed May 14 16:08:08 2025
NAMESPACE: demand-backup-incremental-sharded-13280
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-incremental-sharded-13280.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-incremental-sharded-13280 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-incremental-sharded-13280

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-incremental-sharded-13280 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-incremental-sharded-13280 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K8NWLW68H2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1HfPIdRS3I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K8NWLW68H2
++ cat /tmp/tmp.1HfPIdRS3I
++ rm /tmp/tmp.K8NWLW68H2 /tmp/tmp.1HfPIdRS3I
++ return 0
+ MINIO_POD=minio-service-86dfccd949-9nxqj
+ wait_pod minio-service-86dfccd949-9nxqj
+ local pod=minio-service-86dfccd949-9nxqj
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-9nxqj to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-13280.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.u6JjHWurB1
++ mktemp
+ local LAST_ERR=/tmp/tmp.F8tttUJz5a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-13280.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u6JjHWurB1
service/minio-service created
+ cat /tmp/tmp.F8tttUJz5a
+ rm /tmp/tmp.u6JjHWurB1 /tmp/tmp.F8tttUJz5a
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Xs0bqJwHSY
++ mktemp
+ local LAST_ERR=/tmp/tmp.w3ldWLPDu0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Xs0bqJwHSY
pod "aws-cli" deleted
+ cat /tmp/tmp.w3ldWLPDu0
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.Xs0bqJwHSY /tmp/tmp.w3ldWLPDu0
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tWfukVAtIk
++ mktemp
+ local LAST_ERR=/tmp/tmp.7jKye1D0qE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tWfukVAtIk
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.7jKye1D0qE
+ rm /tmp/tmp.tWfukVAtIk /tmp/tmp.7jKye1D0qE
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ log 'Creating PSMDB cluster'
+ set +o xtrace
[2025-05-14T16:08:52+0000] Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SkpPsJSNuI
++ mktemp
+ local LAST_ERR=/tmp/tmp.MQdTQ7PAu2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SkpPsJSNuI
secret/some-users created
+ cat /tmp/tmp.MQdTQ7PAu2
+ rm /tmp/tmp.SkpPsJSNuI /tmp/tmp.MQdTQ7PAu2
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1904-a4e779d3"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ local LAST_OUT=/tmp/tmp.uasNmTcm50
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.sVXSw6i8Ro
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uasNmTcm50
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.sVXSw6i8Ro
+ rm /tmp/tmp.uasNmTcm50 /tmp/tmp.sVXSw6i8Ro
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.gAuF6yzC0O
++ mktemp
+ local LAST_ERR=/tmp/tmp.49yuSkxbtA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gAuF6yzC0O
deployment.apps/psmdb-client created
+ cat /tmp/tmp.49yuSkxbtA
+ rm /tmp/tmp.gAuF6yzC0O /tmp/tmp.49yuSkxbtA
+ return 0
+ log 'check if all pods started'
+ set +o xtrace
[2025-05-14T16:08:57+0000] check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zT4NGH1aMz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rSLKpPJPoQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zT4NGH1aMz
++ cat /tmp/tmp.rSLKpPJPoQ
++ rm /tmp/tmp.zT4NGH1aMz /tmp/tmp.rSLKpPJPoQ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8WS85owmef
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SE4LgiOpgL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8WS85owmef
++ cat /tmp/tmp.SE4LgiOpgL
++ rm /tmp/tmp.8WS85owmef /tmp/tmp.SE4LgiOpgL
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HGhacM1mS5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oCDYO3oZ1E
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HGhacM1mS5
++ cat /tmp/tmp.oCDYO3oZ1E
++ rm /tmp/tmp.HGhacM1mS5 /tmp/tmp.oCDYO3oZ1E
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mUGD51QehT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fcKa0BXquO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mUGD51QehT
++ cat /tmp/tmp.fcKa0BXquO
++ rm /tmp/tmp.mUGD51QehT /tmp/tmp.fcKa0BXquO
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OwB9vx53dC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.snsumTPBBE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OwB9vx53dC
++ cat /tmp/tmp.snsumTPBBE
++ rm /tmp/tmp.OwB9vx53dC /tmp/tmp.snsumTPBBE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zw6IP0r76H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JtFNI13TAI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zw6IP0r76H
++ cat /tmp/tmp.JtFNI13TAI
++ rm /tmp/tmp.Zw6IP0r76H /tmp/tmp.JtFNI13TAI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ld4hEyC8tZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dn21rqos56
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ld4hEyC8tZ
++ cat /tmp/tmp.dn21rqos56
++ rm /tmp/tmp.ld4hEyC8tZ /tmp/tmp.dn21rqos56
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lRr4w4PdH0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nTNWrmmImT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lRr4w4PdH0
++ cat /tmp/tmp.nTNWrmmImT
++ rm /tmp/tmp.lRr4w4PdH0 /tmp/tmp.nTNWrmmImT
++ return 0
+ lbEndpoint=35.223.172.253
+ '[' -z 35.223.172.253 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-13280
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-13280
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HYk2rHyXyv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0ZsMiXDSbV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HYk2rHyXyv
++ cat /tmp/tmp.0ZsMiXDSbV
++ rm /tmp/tmp.HYk2rHyXyv /tmp/tmp.0ZsMiXDSbV
++ return 0
+ local client_container=psmdb-client-b9788d8bc-5qgs6
+ kubectl_bin exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fDa0fiR3iy
++ mktemp
+ local LAST_ERR=/tmp/tmp.HxDoilxVIK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fDa0fiR3iy
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("760d2522-926c-464c-8ac4-a5f737143dc4") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.HxDoilxVIK
+ rm /tmp/tmp.fDa0fiR3iy /tmp/tmp.HxDoilxVIK
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GrAT31S09y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TSSLJLOAuj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GrAT31S09y
++ cat /tmp/tmp.TSSLJLOAuj
++ rm /tmp/tmp.GrAT31S09y /tmp/tmp.TSSLJLOAuj
++ return 0
+ local client_container=psmdb-client-b9788d8bc-5qgs6
+ kubectl_bin exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jSkk3iASzs
++ mktemp
+ local LAST_ERR=/tmp/tmp.CQJia9HS6g
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jSkk3iASzs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("96f89ef7-35bb-4b59-99e4-2b5a49fa1b0a") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.CQJia9HS6g
+ rm /tmp/tmp.jSkk3iASzs /tmp/tmp.CQJia9HS6g
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BqnhpsOkbS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kcrHZXuOG5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BqnhpsOkbS
++ cat /tmp/tmp.kcrHZXuOG5
++ rm /tmp/tmp.BqnhpsOkbS /tmp/tmp.kcrHZXuOG5
++ return 0
+ local client_container=psmdb-client-b9788d8bc-5qgs6
+ kubectl_bin exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fENzVgDvsy
++ mktemp
+ local LAST_ERR=/tmp/tmp.VarodoNPmV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-5qgs6 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-13280.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fENzVgDvsy
+ cat /tmp/tmp.VarodoNPmV
+ rm /tmp/tmp.fENzVgDvsy /tmp/tmp.VarodoNPmV
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.svtvGSKDf0/find-sharded
+ log 'waiting 60 seconds for stable timestamp in wiredtiger'
+ set +o xtrace
[2025-05-14T16:12:21+0000] waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ log 'running backups'
+ set +o xtrace
[2025-05-14T16:13:41+0000] running backups
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: aws-s3/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.sssNOw4zC0
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gu2EH89cJR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sssNOw4zC0
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.Gu2EH89cJR
+ rm /tmp/tmp.sssNOw4zC0 /tmp/tmp.Gu2EH89cJR
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/'
+ yq '.spec.type="incremental-base"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.a5uyDgugLG
++ mktemp
+ local LAST_ERR=/tmp/tmp.VDTBIS9ulO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a5uyDgugLG
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.VDTBIS9ulO
+ rm /tmp/tmp.a5uyDgugLG /tmp/tmp.VDTBIS9ulO
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ yq '.spec.type="incremental-base"'
+ local LAST_OUT=/tmp/tmp.3kPSNj9YJ0
+ /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/'
++ mktemp
+ local LAST_ERR=/tmp/tmp.HePCLVmKsw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3kPSNj9YJ0
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.HePCLVmKsw
+ rm /tmp/tmp.3kPSNj9YJ0 /tmp/tmp.HePCLVmKsw
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2025-05-14T16:35:11.844Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "760ae18c-2d08-4e83-8a5d-388945b0fdd7"}
2025-05-14T16:35:12.042Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "760ae18c-2d08-4e83-8a5d-388945b0fdd7", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:12.042Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "760ae18c-2d08-4e83-8a5d-388945b0fdd7", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:12.043Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "760ae18c-2d08-4e83-8a5d-388945b0fdd7", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:12.045Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "760ae18c-2d08-4e83-8a5d-388945b0fdd7"}
2025-05-14T16:35:16.593Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5e0d0548-7ba7-41be-bcdc-f9e5b8568592"}
2025-05-14T16:35:16.664Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5e0d0548-7ba7-41be-bcdc-f9e5b8568592", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:16.664Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5e0d0548-7ba7-41be-bcdc-f9e5b8568592", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:16.737Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5e0d0548-7ba7-41be-bcdc-f9e5b8568592", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:16.738Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5e0d0548-7ba7-41be-bcdc-f9e5b8568592"}
2025-05-14T16:35:16.844Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "525d9cdb-26a2-492d-a3ce-ff9120c1be39"}
2025-05-14T16:35:16.914Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "525d9cdb-26a2-492d-a3ce-ff9120c1be39", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:16.914Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "525d9cdb-26a2-492d-a3ce-ff9120c1be39", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:16.914Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "525d9cdb-26a2-492d-a3ce-ff9120c1be39", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:16.916Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "525d9cdb-26a2-492d-a3ce-ff9120c1be39"}
2025-05-14T16:35:17.046Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "186945bc-21e1-4f9b-b3f7-7550a29c75cb"}
2025-05-14T16:35:17.116Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "186945bc-21e1-4f9b-b3f7-7550a29c75cb", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:17.117Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "186945bc-21e1-4f9b-b3f7-7550a29c75cb", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:17.117Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "186945bc-21e1-4f9b-b3f7-7550a29c75cb", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:17.118Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "186945bc-21e1-4f9b-b3f7-7550a29c75cb"}
2025-05-14T16:35:21.739Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "8bc0446a-b58f-43df-9cc5-ab3ac9baa47a"}
2025-05-14T16:35:21.806Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "8bc0446a-b58f-43df-9cc5-ab3ac9baa47a", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:21.806Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "8bc0446a-b58f-43df-9cc5-ab3ac9baa47a", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:21.882Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "8bc0446a-b58f-43df-9cc5-ab3ac9baa47a", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:21.882Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "8bc0446a-b58f-43df-9cc5-ab3ac9baa47a"}
2025-05-14T16:35:21.916Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "ba86f4d1-9c85-48f9-b634-f274b9dfd694"}
2025-05-14T16:35:21.980Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "ba86f4d1-9c85-48f9-b634-f274b9dfd694", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:21.980Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "ba86f4d1-9c85-48f9-b634-f274b9dfd694", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:21.981Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "ba86f4d1-9c85-48f9-b634-f274b9dfd694", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:21.982Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "ba86f4d1-9c85-48f9-b634-f274b9dfd694"}
2025-05-14T16:35:22.118Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "1fb3a35b-c1a2-4214-af44-6d2623b4f08f"}
2025-05-14T16:35:22.188Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "1fb3a35b-c1a2-4214-af44-6d2623b4f08f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:22.188Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "1fb3a35b-c1a2-4214-af44-6d2623b4f08f", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:22.189Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "1fb3a35b-c1a2-4214-af44-6d2623b4f08f", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:22.189Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "1fb3a35b-c1a2-4214-af44-6d2623b4f08f"}
2025-05-14T16:35:26.883Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "6708c6c5-7a8b-401d-896a-853a07eda8af"}
2025-05-14T16:35:26.957Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "6708c6c5-7a8b-401d-896a-853a07eda8af", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:26.958Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "6708c6c5-7a8b-401d-896a-853a07eda8af", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:27.028Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "6708c6c5-7a8b-401d-896a-853a07eda8af", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:27.029Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "6708c6c5-7a8b-401d-896a-853a07eda8af"}
2025-05-14T16:35:27.029Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f3bfb94d-bc3d-442b-aad6-afa9e1761314"}
2025-05-14T16:35:27.097Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f3bfb94d-bc3d-442b-aad6-afa9e1761314", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:27.097Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f3bfb94d-bc3d-442b-aad6-afa9e1761314", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:27.097Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f3bfb94d-bc3d-442b-aad6-afa9e1761314", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:27.098Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f3bfb94d-bc3d-442b-aad6-afa9e1761314"}
2025-05-14T16:35:27.190Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "6816a7fd-276c-4477-ab02-fbc9a0673609"}
2025-05-14T16:35:27.281Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "6816a7fd-276c-4477-ab02-fbc9a0673609", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:27.281Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "6816a7fd-276c-4477-ab02-fbc9a0673609", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:27.281Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "6816a7fd-276c-4477-ab02-fbc9a0673609", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:27.282Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "6816a7fd-276c-4477-ab02-fbc9a0673609"}
2025-05-14T16:35:32.029Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5b50faec-d188-45a1-abe3-bf6f4147df33"}
2025-05-14T16:35:32.101Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5b50faec-d188-45a1-abe3-bf6f4147df33", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:32.101Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5b50faec-d188-45a1-abe3-bf6f4147df33", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:32.172Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5b50faec-d188-45a1-abe3-bf6f4147df33", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:32.174Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "5b50faec-d188-45a1-abe3-bf6f4147df33"}
2025-05-14T16:35:32.174Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "e24aebd0-30ee-4240-a752-6899e66bd521"}
2025-05-14T16:35:32.259Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "e24aebd0-30ee-4240-a752-6899e66bd521", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:32.259Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "e24aebd0-30ee-4240-a752-6899e66bd521", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:32.259Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "e24aebd0-30ee-4240-a752-6899e66bd521", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:32.261Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "e24aebd0-30ee-4240-a752-6899e66bd521"}
2025-05-14T16:35:32.283Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "9bfc0e83-a6ee-411c-9591-92adc12a9b26"}
2025-05-14T16:35:32.358Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "9bfc0e83-a6ee-411c-9591-92adc12a9b26", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:32.358Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "9bfc0e83-a6ee-411c-9591-92adc12a9b26", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:32.358Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "9bfc0e83-a6ee-411c-9591-92adc12a9b26", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:32.359Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "9bfc0e83-a6ee-411c-9591-92adc12a9b26"}
2025-05-14T16:35:37.175Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "e4c27b8f-8e49-4b8f-96cf-25267ede1f7f"}
2025-05-14T16:35:37.254Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "e4c27b8f-8e49-4b8f-96cf-25267ede1f7f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:37.254Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "e4c27b8f-8e49-4b8f-96cf-25267ede1f7f", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:37.333Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "e4c27b8f-8e49-4b8f-96cf-25267ede1f7f", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:37.334Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "e4c27b8f-8e49-4b8f-96cf-25267ede1f7f"}
2025-05-14T16:35:37.334Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "04d79d37-da76-43ba-bb4f-a60b6ad54a7d"}
2025-05-14T16:35:37.414Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "04d79d37-da76-43ba-bb4f-a60b6ad54a7d", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:37.414Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "04d79d37-da76-43ba-bb4f-a60b6ad54a7d", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:37.414Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "04d79d37-da76-43ba-bb4f-a60b6ad54a7d", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:37.415Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "04d79d37-da76-43ba-bb4f-a60b6ad54a7d"}
2025-05-14T16:35:37.415Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f600c372-e61d-444e-95b1-f74190004fd7"}
2025-05-14T16:35:37.492Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f600c372-e61d-444e-95b1-f74190004fd7", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:37.492Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f600c372-e61d-444e-95b1-f74190004fd7", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:37.492Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f600c372-e61d-444e-95b1-f74190004fd7", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:37.493Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f600c372-e61d-444e-95b1-f74190004fd7"}
2025-05-14T16:35:42.335Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "813dfa00-1ad3-4e07-8d32-472c718520c5"}
2025-05-14T16:35:42.413Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "813dfa00-1ad3-4e07-8d32-472c718520c5", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:42.413Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "813dfa00-1ad3-4e07-8d32-472c718520c5", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:42.485Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "813dfa00-1ad3-4e07-8d32-472c718520c5", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:42.486Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "813dfa00-1ad3-4e07-8d32-472c718520c5"}
2025-05-14T16:35:42.486Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a37f3582-d637-4469-91fa-038e91d48abb"}
2025-05-14T16:35:42.555Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a37f3582-d637-4469-91fa-038e91d48abb", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:42.555Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a37f3582-d637-4469-91fa-038e91d48abb", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:42.555Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a37f3582-d637-4469-91fa-038e91d48abb", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:42.556Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a37f3582-d637-4469-91fa-038e91d48abb"}
2025-05-14T16:35:42.556Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "14594c34-058c-4f79-996c-c5778ebffafe"}
2025-05-14T16:35:42.633Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "14594c34-058c-4f79-996c-c5778ebffafe", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:42.633Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "14594c34-058c-4f79-996c-c5778ebffafe", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:42.633Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "14594c34-058c-4f79-996c-c5778ebffafe", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:42.635Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "14594c34-058c-4f79-996c-c5778ebffafe"}
2025-05-14T16:35:47.487Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "711b4147-4d1d-4a37-93d8-31dd159f3124"}
2025-05-14T16:35:47.700Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "711b4147-4d1d-4a37-93d8-31dd159f3124", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:47.700Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "711b4147-4d1d-4a37-93d8-31dd159f3124", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:47.816Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "711b4147-4d1d-4a37-93d8-31dd159f3124", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:47.816Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "711b4147-4d1d-4a37-93d8-31dd159f3124"}
2025-05-14T16:35:47.817Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "5b1b69d2-0186-4f1e-b7b9-7fceff581703"}
2025-05-14T16:35:47.889Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "5b1b69d2-0186-4f1e-b7b9-7fceff581703", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:47.889Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "5b1b69d2-0186-4f1e-b7b9-7fceff581703", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:47.889Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "5b1b69d2-0186-4f1e-b7b9-7fceff581703", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:47.890Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "5b1b69d2-0186-4f1e-b7b9-7fceff581703"}
2025-05-14T16:35:47.890Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f0915d83-6348-43d6-b605-babf5158d84b"}
2025-05-14T16:35:47.966Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f0915d83-6348-43d6-b605-babf5158d84b", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:47.966Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f0915d83-6348-43d6-b605-babf5158d84b", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:47.966Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f0915d83-6348-43d6-b605-babf5158d84b", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:47.967Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "f0915d83-6348-43d6-b605-babf5158d84b"}
2025-05-14T16:35:52.817Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "27c46cee-a370-42c7-868a-7fc3690cda9c"}
2025-05-14T16:35:52.891Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "27c46cee-a370-42c7-868a-7fc3690cda9c", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:52.891Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "27c46cee-a370-42c7-868a-7fc3690cda9c", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:52.979Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "27c46cee-a370-42c7-868a-7fc3690cda9c", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:52.980Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "27c46cee-a370-42c7-868a-7fc3690cda9c"}
2025-05-14T16:35:52.981Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a1444806-64df-44c4-a7d1-20a0706c59ec"}
2025-05-14T16:35:53.058Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a1444806-64df-44c4-a7d1-20a0706c59ec", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:53.058Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a1444806-64df-44c4-a7d1-20a0706c59ec", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:53.058Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a1444806-64df-44c4-a7d1-20a0706c59ec", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:53.059Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "a1444806-64df-44c4-a7d1-20a0706c59ec"}
2025-05-14T16:35:53.060Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "3ffd96c8-be67-4d1c-8dda-09f87a165b5a"}
2025-05-14T16:35:53.132Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "3ffd96c8-be67-4d1c-8dda-09f87a165b5a", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:53.132Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "3ffd96c8-be67-4d1c-8dda-09f87a165b5a", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:53.133Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "3ffd96c8-be67-4d1c-8dda-09f87a165b5a", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:53.134Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "3ffd96c8-be67-4d1c-8dda-09f87a165b5a"}
2025-05-14T16:35:57.981Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "f8a95307-4b17-4f67-8caa-f4125a833e08"}
2025-05-14T16:35:58.053Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "f8a95307-4b17-4f67-8caa-f4125a833e08", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:58.053Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "f8a95307-4b17-4f67-8caa-f4125a833e08", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:35:58.123Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "f8a95307-4b17-4f67-8caa-f4125a833e08", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:35:58.124Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "f8a95307-4b17-4f67-8caa-f4125a833e08"}
2025-05-14T16:35:58.124Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "4cb3723f-82b4-498f-acff-5dda91025611"}
2025-05-14T16:35:58.189Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "4cb3723f-82b4-498f-acff-5dda91025611", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:58.189Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "4cb3723f-82b4-498f-acff-5dda91025611", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:35:58.189Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "4cb3723f-82b4-498f-acff-5dda91025611", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:58.190Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "4cb3723f-82b4-498f-acff-5dda91025611"}
2025-05-14T16:35:58.190Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "d8a89bd0-6ba1-45be-894f-cc39a6d1e2c9"}
2025-05-14T16:35:58.272Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "d8a89bd0-6ba1-45be-894f-cc39a6d1e2c9", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:35:58.272Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "d8a89bd0-6ba1-45be-894f-cc39a6d1e2c9", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:35:58.272Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "d8a89bd0-6ba1-45be-894f-cc39a6d1e2c9", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:35:58.273Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "d8a89bd0-6ba1-45be-894f-cc39a6d1e2c9"}
2025-05-14T16:36:03.125Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "286171a9-9a40-433a-8e38-0fbb291009e8"}
2025-05-14T16:36:03.193Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "286171a9-9a40-433a-8e38-0fbb291009e8", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:03.193Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "286171a9-9a40-433a-8e38-0fbb291009e8", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:36:03.279Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "286171a9-9a40-433a-8e38-0fbb291009e8", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:36:03.281Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "286171a9-9a40-433a-8e38-0fbb291009e8"}
2025-05-14T16:36:03.281Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f0944fd9-a538-4518-bf0b-b879e5e2bd4f"}
2025-05-14T16:36:03.355Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f0944fd9-a538-4518-bf0b-b879e5e2bd4f", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:03.355Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f0944fd9-a538-4518-bf0b-b879e5e2bd4f", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:36:03.355Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f0944fd9-a538-4518-bf0b-b879e5e2bd4f", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:03.357Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "f0944fd9-a538-4518-bf0b-b879e5e2bd4f"}
2025-05-14T16:36:03.357Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "8b153f26-9add-4c40-902c-5f1502b634bf"}
2025-05-14T16:36:03.435Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "8b153f26-9add-4c40-902c-5f1502b634bf", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:03.435Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "8b153f26-9add-4c40-902c-5f1502b634bf", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:36:03.435Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "8b153f26-9add-4c40-902c-5f1502b634bf", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:03.436Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "8b153f26-9add-4c40-902c-5f1502b634bf"}
2025-05-14T16:36:08.281Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "ede72233-7de0-4959-a15e-3afc83e98654"}
2025-05-14T16:36:08.369Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "ede72233-7de0-4959-a15e-3afc83e98654", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:08.369Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "ede72233-7de0-4959-a15e-3afc83e98654", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:36:08.466Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "ede72233-7de0-4959-a15e-3afc83e98654", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:36:08.467Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "ede72233-7de0-4959-a15e-3afc83e98654"}
2025-05-14T16:36:08.467Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "b54dc22e-233d-44b5-a3bd-0684c1d93c4a"}
2025-05-14T16:36:08.536Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "b54dc22e-233d-44b5-a3bd-0684c1d93c4a", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:08.536Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "b54dc22e-233d-44b5-a3bd-0684c1d93c4a", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:36:08.536Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "b54dc22e-233d-44b5-a3bd-0684c1d93c4a", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:08.537Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "b54dc22e-233d-44b5-a3bd-0684c1d93c4a"}
2025-05-14T16:36:08.537Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "13971b7a-e849-4093-983a-476c95bd3529"}
2025-05-14T16:36:08.611Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "13971b7a-e849-4093-983a-476c95bd3529", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:08.611Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "13971b7a-e849-4093-983a-476c95bd3529", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:36:08.611Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "13971b7a-e849-4093-983a-476c95bd3529", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:08.612Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "13971b7a-e849-4093-983a-476c95bd3529"}
2025-05-14T16:36:13.467Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "906c7bd7-50a9-4280-a297-b9b94d817be9"}
2025-05-14T16:36:13.542Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "906c7bd7-50a9-4280-a297-b9b94d817be9", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:13.542Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "906c7bd7-50a9-4280-a297-b9b94d817be9", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:36:13.612Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "906c7bd7-50a9-4280-a297-b9b94d817be9", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:36:13.613Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "906c7bd7-50a9-4280-a297-b9b94d817be9"}
2025-05-14T16:36:13.613Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "d9f331c9-2db5-4a7d-b707-2e14af59a462"}
2025-05-14T16:36:13.684Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "d9f331c9-2db5-4a7d-b707-2e14af59a462", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:13.684Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "d9f331c9-2db5-4a7d-b707-2e14af59a462", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:36:13.685Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "d9f331c9-2db5-4a7d-b707-2e14af59a462", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:13.686Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "d9f331c9-2db5-4a7d-b707-2e14af59a462"}
2025-05-14T16:36:13.686Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "ee7230f2-fe8d-4b31-976c-bec8a39579fe"}
2025-05-14T16:36:13.756Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "ee7230f2-fe8d-4b31-976c-bec8a39579fe", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:13.756Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "ee7230f2-fe8d-4b31-976c-bec8a39579fe", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:36:13.756Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "ee7230f2-fe8d-4b31-976c-bec8a39579fe", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:13.757Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "ee7230f2-fe8d-4b31-976c-bec8a39579fe"}
2025-05-14T16:36:18.614Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "45f025a7-cfe7-4f50-a664-62e31d19031e"}
2025-05-14T16:36:18.691Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "45f025a7-cfe7-4f50-a664-62e31d19031e", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:18.691Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "45f025a7-cfe7-4f50-a664-62e31d19031e", "currentJob": {"Name":"backup-azure-blob-sharded","Type":0}}
2025-05-14T16:36:18.761Z	INFO	Waiting for backup metadata	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "45f025a7-cfe7-4f50-a664-62e31d19031e", "pbmName": "2025-05-14T16:13:49Z", "backup": "backup-azure-blob-sharded"}
2025-05-14T16:36:18.762Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-azure-blob-sharded", "reconcileID": "45f025a7-cfe7-4f50-a664-62e31d19031e"}
2025-05-14T16:36:18.762Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "42289fa2-faef-4a7f-a07d-c624a0665aa7"}
2025-05-14T16:36:18.838Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "42289fa2-faef-4a7f-a07d-c624a0665aa7", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:18.838Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "42289fa2-faef-4a7f-a07d-c624a0665aa7", "currentJob": {"Name":"backup-gcp-cs-sharded","Type":0}}
2025-05-14T16:36:18.838Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "42289fa2-faef-4a7f-a07d-c624a0665aa7", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:18.839Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-gcp-cs-sharded", "reconcileID": "42289fa2-faef-4a7f-a07d-c624a0665aa7"}
2025-05-14T16:36:18.839Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "61b4f318-6585-43de-a61f-6973e6621dcc"}
2025-05-14T16:36:18.909Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "61b4f318-6585-43de-a61f-6973e6621dcc", "cluster": "some-name", "namespace": "demand-backup-incremental-sharded-13280"}
2025-05-14T16:36:18.909Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "61b4f318-6585-43de-a61f-6973e6621dcc", "currentJob": {"Name":"backup-aws-s3-sharded","Type":0}}
2025-05-14T16:36:18.909Z	INFO	Waiting for backup to complete	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "61b4f318-6585-43de-a61f-6973e6621dcc", "backup": "backup-azure-blob-sharded", "status": "requested"}
2025-05-14T16:36:18.910Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3-sharded","namespace":"demand-backup-incremental-sharded-13280"}, "namespace": "demand-backup-incremental-sharded-13280", "name": "backup-aws-s3-sharded", "reconcileID": "61b4f318-6585-43de-a61f-6973e6621dcc"}
NAME                        CLUSTER     STORAGE      DESTINATION                                                                             TYPE               STATUS      COMPLETED   AGE
backup-aws-s3-sharded       some-name   aws-s3       s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-14T16:13:44Z      incremental-base   waiting                 22m
backup-azure-blob-sharded   some-name   azure-blob   azure://operator-testing/psmdb-demand-backup-incremental-sharded/2025-05-14T16:13:49Z                      requested               22m
backup-gcp-cs-sharded       some-name   gcp-cs                                                                                                                  waiting                 22m
Backup object psmdb-backup/backup-aws-s3-sharded is in waiting state.
something went wrong with operator or kubernetes cluster