Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/demand-backup-incremental-gcp-s3.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + '[' '!' -z '' ']' + create_infra demand-backup-incremental-gcp-s3-25718 + local ns=demand-backup-incremental-gcp-s3-25718 + echo demand-backup-incremental-gcp-s3-25718 + [[ 0 == 1 ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.EfXRj3SZ2M + local LAST_OUT=/tmp/tmp.wW0Lo56xPA ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.83ajBMrfd9 + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.BrPYEiyu87 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EfXRj3SZ2M + cat /tmp/tmp.83ajBMrfd9 + rm /tmp/tmp.EfXRj3SZ2M /tmp/tmp.83ajBMrfd9 + return 0 namespace "demand-backup-incremental-gcp-native-23181" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wW0Lo56xPA namespace "psmdb-operator" deleted + cat /tmp/tmp.BrPYEiyu87 + rm /tmp/tmp.wW0Lo56xPA /tmp/tmp.BrPYEiyu87 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.3rgBKmrFLp ++ mktemp + local LAST_ERR=/tmp/tmp.ERsibO7oiD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3rgBKmrFLp + cat /tmp/tmp.ERsibO7oiD + rm /tmp/tmp.3rgBKmrFLp /tmp/tmp.ERsibO7oiD + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.pOeLtzbtUE ++ mktemp + local LAST_ERR=/tmp/tmp.8ETXNyhqYI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pOeLtzbtUE namespace/psmdb-operator created + cat /tmp/tmp.8ETXNyhqYI + rm /tmp/tmp.pOeLtzbtUE /tmp/tmp.8ETXNyhqYI + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.cBZ1SI7u43 +++ mktemp ++ local LAST_ERR=/tmp/tmp.p4MM01kmO1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cBZ1SI7u43 ++ cat /tmp/tmp.p4MM01kmO1 ++ rm /tmp/tmp.cBZ1SI7u43 /tmp/tmp.p4MM01kmO1 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.sS8q6G3xnq ++ mktemp + local LAST_ERR=/tmp/tmp.c9D6H0AdAM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sS8q6G3xnq Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11" modified. + cat /tmp/tmp.c9D6H0AdAM + rm /tmp/tmp.sS8q6G3xnq /tmp/tmp.c9D6H0AdAM + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.3b36q6B8kq ++ mktemp + local LAST_ERR=/tmp/tmp.uloKYo4e2N + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3b36q6B8kq customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.uloKYo4e2N + rm /tmp/tmp.3b36q6B8kq /tmp/tmp.uloKYo4e2N + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9PXwg397vl ++ mktemp + local LAST_ERR=/tmp/tmp.Tvco9GP1Ml + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9PXwg397vl clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged + cat /tmp/tmp.Tvco9GP1Ml + rm /tmp/tmp.9PXwg397vl /tmp/tmp.Tvco9GP1Ml + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.IRk9QDP5NB ++ mktemp + local LAST_ERR=/tmp/tmp.60JN6vWSEf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IRk9QDP5NB deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.60JN6vWSEf + rm /tmp/tmp.IRk9QDP5NB /tmp/tmp.60JN6vWSEf + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.LRZFRhi3kQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ycozRaG8LO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LRZFRhi3kQ ++ cat /tmp/tmp.ycozRaG8LO ++ rm /tmp/tmp.LRZFRhi3kQ /tmp/tmp.ycozRaG8LO ++ return 0 + wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-bv4qr + local pod=percona-server-mongodb-operator-859f4bc6d7-bv4qr + set +o xtrace waiting for pod/percona-server-mongodb-operator-859f4bc6d7-bv4qr to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Psi3gl7o9H +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gx3kVsCuXQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Psi3gl7o9H ++ cat /tmp/tmp.Gx3kVsCuXQ ++ rm /tmp/tmp.Psi3gl7o9H /tmp/tmp.Gx3kVsCuXQ ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-bv4qr ++ mktemp + local LAST_OUT=/tmp/tmp.TyUiZOnNHk ++ mktemp + local LAST_ERR=/tmp/tmp.gl1QVDU35r + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-bv4qr + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TyUiZOnNHk + cat /tmp/tmp.gl1QVDU35r + rm /tmp/tmp.TyUiZOnNHk /tmp/tmp.gl1QVDU35r + return 0 2026-06-09T07:41:05.482Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-incremental-gcp-s3-25718 + local namespace=demand-backup-incremental-gcp-s3-25718 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces demand-backup-incremental-gcp-s3-25718' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-incremental-gcp-s3-25718 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-incremental-gcp-s3-25718 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.8oOv0xnvAs ++ mktemp + local LAST_OUT=/tmp/tmp.V3Nse12dG7 ++ mktemp + local LAST_ERR=/tmp/tmp.gnnKXG5gEM + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.0hSW9kbE7H + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-incremental-gcp-s3-25718 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8oOv0xnvAs + cat /tmp/tmp.gnnKXG5gEM + rm /tmp/tmp.8oOv0xnvAs /tmp/tmp.gnnKXG5gEM + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V3Nse12dG7 + cat /tmp/tmp.0hSW9kbE7H + rm /tmp/tmp.V3Nse12dG7 /tmp/tmp.0hSW9kbE7H + return 0 + kubectl_bin wait --for=delete namespace demand-backup-incremental-gcp-s3-25718 ++ mktemp + local LAST_OUT=/tmp/tmp.35OarS6AO7 ++ mktemp + local LAST_ERR=/tmp/tmp.t8tJAFG6P9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-incremental-gcp-s3-25718 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.35OarS6AO7 + cat /tmp/tmp.t8tJAFG6P9 + rm /tmp/tmp.35OarS6AO7 /tmp/tmp.t8tJAFG6P9 + return 0 + desc 'create namespace demand-backup-incremental-gcp-s3-25718' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-incremental-gcp-s3-25718 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-incremental-gcp-s3-25718 ++ mktemp + local LAST_OUT=/tmp/tmp.Y7haa2By95 ++ mktemp + local LAST_ERR=/tmp/tmp.3g0Mw0jhgD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-incremental-gcp-s3-25718 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y7haa2By95 namespace/demand-backup-incremental-gcp-s3-25718 created + cat /tmp/tmp.3g0Mw0jhgD + rm /tmp/tmp.Y7haa2By95 /tmp/tmp.3g0Mw0jhgD + return 0 + set_kube_ctx demand-backup-incremental-gcp-s3-25718 + local namespace=demand-backup-incremental-gcp-s3-25718 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.naMf4mSNaz +++ mktemp ++ local LAST_ERR=/tmp/tmp.pdizdgBTpz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.naMf4mSNaz ++ cat /tmp/tmp.pdizdgBTpz ++ rm /tmp/tmp.naMf4mSNaz /tmp/tmp.pdizdgBTpz ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=demand-backup-incremental-gcp-s3-25718 ++ mktemp + local LAST_OUT=/tmp/tmp.blAT1XFO6S ++ mktemp + local LAST_ERR=/tmp/tmp.p8kfI3MBG2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=demand-backup-incremental-gcp-s3-25718 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.blAT1XFO6S Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11" modified. + cat /tmp/tmp.p8kfI3MBG2 + rm /tmp/tmp.blAT1XFO6S /tmp/tmp.p8kfI3MBG2 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.TJIo0icFcl ++ mktemp + local LAST_ERR=/tmp/tmp.QDv8CWzyl0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TJIo0icFcl secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.QDv8CWzyl0 + rm /tmp/tmp.TJIo0icFcl /tmp/tmp.QDv8CWzyl0 + return 0 + desc 'Testing on not sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on not sharded cluster ----------------------------------------------------------------------------------- + log 'Creating PSMDB cluster' + set +o xtrace [2026-06-09T07:41:44+0000] Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.NhdwA2cQnv ++ mktemp + local LAST_ERR=/tmp/tmp.UW5TtKxtoK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NhdwA2cQnv secret/some-users created + cat /tmp/tmp.UW5TtKxtoK + rm /tmp/tmp.NhdwA2cQnv /tmp/tmp.UW5TtKxtoK + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.BFtlAdz61V ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-incremental-gcp-s3-25718/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.KZ6rgtj6qF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BFtlAdz61V perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.KZ6rgtj6qF + rm /tmp/tmp.BFtlAdz61V /tmp/tmp.KZ6rgtj6qF + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.BZCRTON8m6 ++ mktemp + local LAST_ERR=/tmp/tmp.Oq7zISx923 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BZCRTON8m6 deployment.apps/psmdb-client created + cat /tmp/tmp.Oq7zISx923 + rm /tmp/tmp.BZCRTON8m6 /tmp/tmp.Oq7zISx923 + return 0 + log 'check if all pods started' + set +o xtrace [2026-06-09T07:41:50+0000] check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FueB9TTJsI +++ mktemp ++ local LAST_ERR=/tmp/tmp.aA4ktoEDcC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FueB9TTJsI ++ cat /tmp/tmp.aA4ktoEDcC ++ rm /tmp/tmp.FueB9TTJsI /tmp/tmp.aA4ktoEDcC ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lMgUQQhdIv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZXstFwpK9j ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lMgUQQhdIv ++ cat /tmp/tmp.ZXstFwpK9j ++ rm /tmp/tmp.lMgUQQhdIv /tmp/tmp.ZXstFwpK9j ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CPHDEfQZ34 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FzEFu2PMIg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CPHDEfQZ34 ++ cat /tmp/tmp.FzEFu2PMIg ++ rm /tmp/tmp.CPHDEfQZ34 /tmp/tmp.FzEFu2PMIg ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MYhyH3cJWz +++ mktemp ++ local LAST_ERR=/tmp/tmp.SFQtn0xrHY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MYhyH3cJWz ++ cat /tmp/tmp.SFQtn0xrHY ++ rm /tmp/tmp.MYhyH3cJWz /tmp/tmp.SFQtn0xrHY ++ return 0 + [[ ready == ready ]] + echo .OK .OK + log 'writing test data' + set +o xtrace [2026-06-09T07:43:48+0000] writing test data + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-incremental-gcp-s3-25718 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4xsuwKeXdi +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kqc9eUrCkN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4xsuwKeXdi ++ cat /tmp/tmp.Kqc9eUrCkN ++ rm /tmp/tmp.4xsuwKeXdi /tmp/tmp.Kqc9eUrCkN ++ return 0 + local client_container=psmdb-client-5649fbb65f-9wnnl + kubectl_bin exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.NR2Pxu2JIi ++ mktemp + local LAST_ERR=/tmp/tmp.Sir8wOjhYz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NR2Pxu2JIi Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("e9b79bc9-4dfe-421e-bf93-6506d467ee5c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.Sir8wOjhYz + rm /tmp/tmp.NR2Pxu2JIi /tmp/tmp.Sir8wOjhYz + return 0 + sleep 1 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-incremental-gcp-s3-25718 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IwtphcjaY5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gh5VJVVLud ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IwtphcjaY5 ++ cat /tmp/tmp.gh5VJVVLud ++ rm /tmp/tmp.IwtphcjaY5 /tmp/tmp.gh5VJVVLud ++ return 0 + local client_container=psmdb-client-5649fbb65f-9wnnl + kubectl_bin exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.n4X1hMuXso ++ mktemp + local LAST_ERR=/tmp/tmp.Gsx91ifFcl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.n4X1hMuXso Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f40edf52-bc21-4f92-a4a0-cf02d64c3f17") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Gsx91ifFcl + rm /tmp/tmp.n4X1hMuXso /tmp/tmp.Gsx91ifFcl + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T07:44:00+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n5XQPnKSVc +++ mktemp ++ local LAST_ERR=/tmp/tmp.7QHJYeRfU6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.n5XQPnKSVc ++ cat /tmp/tmp.7QHJYeRfU6 ++ rm /tmp/tmp.n5XQPnKSVc /tmp/tmp.7QHJYeRfU6 ++ return 0 + local client_container=psmdb-client-5649fbb65f-9wnnl + kubectl_bin exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.1124GU2vB0 ++ mktemp + local LAST_ERR=/tmp/tmp.pov45Urcdm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1124GU2vB0 + cat /tmp/tmp.pov45Urcdm + rm /tmp/tmp.1124GU2vB0 /tmp/tmp.pov45Urcdm + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/compare/find.json /tmp/tmp.iUOpZ1cC75/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T07:44:02+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' +++ mktemp + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local LAST_OUT=/tmp/tmp.HRqOonk1Ix +++ mktemp ++ local LAST_ERR=/tmp/tmp.gWdhAQhceK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HRqOonk1Ix ++ cat /tmp/tmp.gWdhAQhceK ++ rm /tmp/tmp.HRqOonk1Ix /tmp/tmp.gWdhAQhceK ++ return 0 + local client_container=psmdb-client-5649fbb65f-9wnnl + kubectl_bin exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.XoHmQHO5cF ++ mktemp + local LAST_ERR=/tmp/tmp.WMB0OgrIvg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XoHmQHO5cF + cat /tmp/tmp.WMB0OgrIvg + rm /tmp/tmp.XoHmQHO5cF /tmp/tmp.WMB0OgrIvg + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/compare/find.json /tmp/tmp.iUOpZ1cC75/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T07:44:05+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BJDCsjB5FI +++ mktemp ++ local LAST_ERR=/tmp/tmp.df0X6wudfF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BJDCsjB5FI ++ cat /tmp/tmp.df0X6wudfF ++ rm /tmp/tmp.BJDCsjB5FI /tmp/tmp.df0X6wudfF ++ return 0 + local client_container=psmdb-client-5649fbb65f-9wnnl + kubectl_bin exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Dq5OihI0ak ++ mktemp + local LAST_ERR=/tmp/tmp.lc32PddywO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-9wnnl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Dq5OihI0ak + cat /tmp/tmp.lc32PddywO + rm /tmp/tmp.Dq5OihI0ak /tmp/tmp.lc32PddywO + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/compare/find.json /tmp/tmp.iUOpZ1cC75/find + log 'running backup' + set +o xtrace [2026-06-09T07:44:08+0000] running backup + backup_name_gcp_cs_s3=backup-gcp-cs-s3 + run_backup gcp-cs-s3 backup-gcp-cs-s3 incremental-base + local storage=gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local type=incremental-base + log 'running backup backup-gcp-cs-s3' + set +o xtrace [2026-06-09T07:44:08+0000] running backup backup-gcp-cs-s3 + yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "incremental-base"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-incremental-gcp-s3/conf/backup-gcp-cs-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TM2lVKsDNa ++ mktemp + local LAST_ERR=/tmp/tmp.7WkkWjzXPL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TM2lVKsDNa perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created + cat /tmp/tmp.7WkkWjzXPL + rm /tmp/tmp.TM2lVKsDNa /tmp/tmp.7WkkWjzXPL + return 0 + wait_backup backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-s3 to reach ready state......................................................................2026-06-09T07:41:50.903Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "b445e696-69ba-4196-8a72-91be582601b2", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:41:50.903Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "b445e696-69ba-4196-8a72-91be582601b2", "previous": "", "current": "initializing"} 2026-06-09T07:41:51.890Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90642270-35d6-4c81-8c0f-e587598d3762", "replset": "rs0", "size": 3, "pods": 1} 2026-06-09T07:41:51.890Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90642270-35d6-4c81-8c0f-e587598d3762", "replset": "rs0", "status": "initializing"} 2026-06-09T07:41:52.195Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90642270-35d6-4c81-8c0f-e587598d3762", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:41:53.168Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "b1c078bf-b0d3-4a6a-9049-6fb42fae6ac0", "replset": "rs0", "size": 3, "pods": 1} 2026-06-09T07:41:53.168Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "b1c078bf-b0d3-4a6a-9049-6fb42fae6ac0", "replset": "rs0", "status": "initializing"} 2026-06-09T07:41:53.481Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "b1c078bf-b0d3-4a6a-9049-6fb42fae6ac0", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:41:59.723Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "9fa5b7fb-595c-4198-a082-1445c0cd46c5", "replset": "rs0", "size": 3, "pods": 1} 2026-06-09T07:41:59.723Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "9fa5b7fb-595c-4198-a082-1445c0cd46c5", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:00.036Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "9fa5b7fb-595c-4198-a082-1445c0cd46c5", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:06.272Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "65681d6b-11a9-411a-b839-fdb9da0a29c1", "replset": "rs0", "size": 3, "pods": 1} 2026-06-09T07:42:06.272Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "65681d6b-11a9-411a-b839-fdb9da0a29c1", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:06.585Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "65681d6b-11a9-411a-b839-fdb9da0a29c1", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:12.589Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "95e51fdd-f90c-4e2f-9378-ff96112bf1f4", "replset": "rs0", "size": 3, "pods": 1} 2026-06-09T07:42:12.589Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "95e51fdd-f90c-4e2f-9378-ff96112bf1f4", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:12.907Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "95e51fdd-f90c-4e2f-9378-ff96112bf1f4", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:18.942Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "3022540a-6f1e-481c-9973-f4a8aaac0d65", "replset": "rs0", "size": 3, "pods": 2} 2026-06-09T07:42:18.943Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "3022540a-6f1e-481c-9973-f4a8aaac0d65", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:19.262Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "3022540a-6f1e-481c-9973-f4a8aaac0d65", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:20.363Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "efc06b75-5e3d-4d8c-a857-10712956ed1a", "replset": "rs0", "size": 3, "pods": 2} 2026-06-09T07:42:20.363Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "efc06b75-5e3d-4d8c-a857-10712956ed1a", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:20.783Z INFO PBM pbm-agent version {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "efc06b75-5e3d-4d8c-a857-10712956ed1a", "pod": "some-name-rs0-0", "image": "docker.io/perconalab/percona-server-mongodb-operator:main-backup", "version": "2.14.0"} 2026-06-09T07:42:20.804Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "efc06b75-5e3d-4d8c-a857-10712956ed1a", "replset": "rs0"} 2026-06-09T07:42:20.970Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "efc06b75-5e3d-4d8c-a857-10712956ed1a", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:21.983Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "acef5413-c34e-44c8-81c4-29b8a2d6b1ad", "replset": "rs0", "size": 3, "pods": 2} 2026-06-09T07:42:21.983Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "acef5413-c34e-44c8-81c4-29b8a2d6b1ad", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:22.226Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "acef5413-c34e-44c8-81c4-29b8a2d6b1ad", "replset": "rs0"} 2026-06-09T07:42:22.397Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "acef5413-c34e-44c8-81c4-29b8a2d6b1ad", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:28.916Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90938576-e0fe-4186-98e3-7fd9cac7b15e", "replset": "rs0", "size": 3, "pods": 2} 2026-06-09T07:42:28.916Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90938576-e0fe-4186-98e3-7fd9cac7b15e", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:29.150Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90938576-e0fe-4186-98e3-7fd9cac7b15e", "replset": "rs0"} 2026-06-09T07:42:29.305Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "90938576-e0fe-4186-98e3-7fd9cac7b15e", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:35.269Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "bd8fa8c3-3571-45bb-97c0-c73dbb68336a", "replset": "rs0", "size": 3, "pods": 2} 2026-06-09T07:42:35.269Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "bd8fa8c3-3571-45bb-97c0-c73dbb68336a", "replset": "rs0", "status": "initializing"} 2026-06-09T07:42:35.510Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "bd8fa8c3-3571-45bb-97c0-c73dbb68336a", "replset": "rs0"} 2026-06-09T07:42:35.671Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "bd8fa8c3-3571-45bb-97c0-c73dbb68336a", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-06-09T07:42:51.804Z INFO initiating replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0", "pod": "some-name-rs0-0"} 2026-06-09T07:42:58.003Z INFO some-name-rs0-0 is the writable primary {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0"} 2026-06-09T07:42:58.004Z INFO replset initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0", "pod": "some-name-rs0-0"} 2026-06-09T07:42:59.332Z DEBUG userAdmin authentication failed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "pod": "some-name-rs0-0", "error": "command terminated with exit code 1", "stdout": "", "stderr": "MongoServerError: Authentication failed.\n"} 2026-06-09T07:42:59.332Z INFO creating user admin {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2026-06-09T07:43:01.804Z INFO user admin created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2026-06-09T07:43:01.974Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "database": "admin", "user": "clusterAdmin"} 2026-06-09T07:43:02.031Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "database": "admin", "user": "clusterMonitor"} 2026-06-09T07:43:02.129Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "database": "admin", "user": "backup$#%"} 2026-06-09T07:43:02.226Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "database": "admin", "user": "databaseAdmin"} 2026-06-09T07:43:02.302Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "replset": "rs0", "status": "initializing"} 2026-06-09T07:43:02.636Z INFO PBM configuration changed or resync is needed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "oldHash": "", "newHash": "bda9e8f74dd6a08b99d9a6fddba776a3b62c3b5a70b23b6edea49dc95c3fde89"} 2026-06-09T07:43:02.653Z INFO PBM Setting config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "cluster": "some-name", "mainStorage": "gcp-cs-s3"} 2026-06-09T07:43:02.709Z INFO PBM waiting for resync to start (up to 30 seconds) {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5"} 2026-06-09T07:43:32.761Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "314146f1-7b03-44f5-a0f7-1a32e320dbc5", "error": "reconcile PBM: reconcile configuration: resync: wait for resync: wait for resync to start: resync is not started until deadline", "errorVerbose": "resync is not started until deadline\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1229\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1244\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nwait for resync to start\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1245\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nwait for resync\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1210\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nresync\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:182\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile configuration\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile PBM\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:518\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:43:34.073Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "_id": 1, "host": "some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017"} 2026-06-09T07:43:34.073Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "replset": "rs0"} 2026-06-09T07:43:34.073Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "cfg": {"_id":"rs0","version":2,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-rxh7","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-wrcn","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a27c3fe4e727708a121d0a5"},"writeConcernMajorityJournalDefault":true}} 2026-06-09T07:43:34.090Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "replset": "rs0"} 2026-06-09T07:43:34.090Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "cfg": {"_id":"rs0","version":3,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-rxh7","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-wrcn","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":0}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a27c3fe4e727708a121d0a5"},"writeConcernMajorityJournalDefault":true}} 2026-06-09T07:43:34.099Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "liveMembers": 1, "pods": 3} 2026-06-09T07:43:34.100Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "replset": "rs0", "status": "initializing"} 2026-06-09T07:43:34.473Z INFO PBM configuration changed or resync is needed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "oldHash": "", "newHash": "bda9e8f74dd6a08b99d9a6fddba776a3b62c3b5a70b23b6edea49dc95c3fde89"} 2026-06-09T07:43:34.491Z INFO PBM Setting config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "cluster": "some-name", "mainStorage": "gcp-cs-s3"} 2026-06-09T07:43:34.795Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"} 2026-06-09T07:43:34.796Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "6c52e11c-b82d-44e8-9bbf-2c92cb455167", "previous": "error", "current": "initializing"} 2026-06-09T07:43:36.232Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "_id": 2, "host": "some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017"} 2026-06-09T07:43:36.232Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "replset": "rs0"} 2026-06-09T07:43:36.232Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "cfg": {"_id":"rs0","version":4,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-rxh7","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-wrcn","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":0},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-x259","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a27c3fe4e727708a121d0a5"},"writeConcernMajorityJournalDefault":true}} 2026-06-09T07:43:36.248Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "replset": "rs0"} 2026-06-09T07:43:36.248Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "cfg": {"_id":"rs0","version":5,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-rxh7","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-wrcn","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.demand-backup-incremental-gcp-s3-25718.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2058-fb8cc-default-pool-f8967f57-x259","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a27c3fe4e727708a121d0a5"},"writeConcernMajorityJournalDefault":true}} 2026-06-09T07:43:36.255Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "liveMembers": 1, "pods": 3} 2026-06-09T07:43:36.256Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "replset": "rs0", "status": "initializing"} 2026-06-09T07:43:36.819Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "eee708c6-a37a-4753-bcc8-e5312625aa32", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"} 2026-06-09T07:43:38.190Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "4b0803d7-732b-40be-9873-0635bc97c413", "liveMembers": 1, "pods": 3} 2026-06-09T07:43:38.192Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "4b0803d7-732b-40be-9873-0635bc97c413", "replset": "rs0", "status": "initializing"} 2026-06-09T07:43:38.779Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "4b0803d7-732b-40be-9873-0635bc97c413", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"} 2026-06-09T07:43:40.665Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "23c4855d-d7dc-4d23-96b1-8b6f68489c38", "previous": "initializing", "current": "ready"} 2026-06-09T07:43:41.628Z INFO update Mongo version to 8.0.23-10 (fetched from db) {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "some-name", "reconcileID": "30a64513-e835-43f0-8230-e3afc4da9a1b"} 2026-06-09T07:46:32.074Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e", "backup": "backup-gcp-cs-native", "error": "create backup object: create pbm object: get mongo uri: get secrets: Secret \"internal-some-name-users\" not found", "errorVerbose": "Secret \"internal-some-name-users\" not found\nget secrets\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.getMongoUri\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:216\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:53\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nget mongo uri\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:218\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:53\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ncreate pbm object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:55\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ncreate backup object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:214\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:148 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.075Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e", "previous": "ready", "current": "error"} 2026-06-09T07:46:32.108Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e", "lease": "psmdb-some-name-backup-lock"} 2026-06-09T07:46:32.209Z ERROR failed to release the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e", "error": "get lease: Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found", "errorVerbose": "Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found\nget lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.ReleaseLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:66\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:162\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:164 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.209Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e"} 2026-06-09T07:46:32.209Z INFO Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e"} 2026-06-09T07:46:32.209Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "1fa2c2cb-273b-4342-9258-e3b8a980286e", "error": "create backup object: create pbm object: get mongo uri: get secrets: Secret \"internal-some-name-users\" not found", "errorVerbose": "Secret \"internal-some-name-users\" not found\nget secrets\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.getMongoUri\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:216\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:53\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nget mongo uri\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.NewPBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:218\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:53\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ncreate pbm object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).newManagedBackups\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/backup.go:55\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:212\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\ncreate backup object\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func3\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:214\nk8s.io/client-go/util/retry.OnError.func1\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/go/pkg/mod/k8s.io/client-go@v0.36.1/util/retry/util.go:50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:202\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.209Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9"} 2026-06-09T07:46:32.222Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "state": ""} 2026-06-09T07:46:32.223Z INFO Backup didn't start in startingDeadlineSeconds, failing the backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "startingDeadlineSeconds": 120, "passedSeconds": 142.223047565} 2026-06-09T07:46:32.223Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "backup": "backup-gcp-cs-s3", "error": "starting deadline seconds exceeded", "errorVerbose": "starting deadline seconds exceeded\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.checkStartingDeadline\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/deadline.go:40\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:184\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:148 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.223Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "previous": "", "current": "error"} 2026-06-09T07:46:32.249Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "lease": "psmdb-some-name-backup-lock"} 2026-06-09T07:46:32.249Z ERROR failed to release the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "error": "get lease: Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found", "errorVerbose": "Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found\nget lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.ReleaseLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:66\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:162\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:164 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.249Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9"} 2026-06-09T07:46:32.249Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "7943e4b2-ebc8-4266-af4b-efaf4de5f1d9", "error": "starting deadline seconds exceeded", "errorVerbose": "starting deadline seconds exceeded\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.checkStartingDeadline\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/deadline.go:40\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:184\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.250Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba"} 2026-06-09T07:46:32.263Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba", "state": "error"} 2026-06-09T07:46:32.264Z INFO releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba", "lease": "psmdb-some-name-backup-lock", "holder": "backup-gcp-cs-native-6130a4ff-76b8-4907-842d-c2c9b963a513"} 2026-06-09T07:46:32.264Z DEBUG failed to release backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba", "lease": "psmdb-some-name-backup-lock", "holder": "backup-gcp-cs-native-6130a4ff-76b8-4907-842d-c2c9b963a513", "error": "get lease: Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found", "errorVerbose": "Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found\nget lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.ReleaseLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:66\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).runReleaseLockFinalizer\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:569\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).checkFinalizers\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:549\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} 2026-06-09T07:46:32.297Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba"} 2026-06-09T07:46:32.297Z ERROR Observed a panic {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba", "panic": "runtime error: invalid memory address or nil pointer dereference", "panicGoValue": "\"invalid memory address or nil pointer dereference\"", "stacktrace": "goroutine 207 [running]:\nk8s.io/apimachinery/pkg/util/runtime.logPanic({0x3673528, 0x1f419a212840}, {0x2e23a20, 0x56240e0})\n\t/go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/runtime/runtime.go:134 +0xbc\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile.func1()\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:201 +0x103\npanic({0x2e23a20?, 0x56240e0?})\n\t/usr/local/go/src/runtime/panic.go:860 +0x13a\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func4()\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:223 +0x50\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile(0x1f41999cd940, {0x3673528, 0x1f419a212840}, {{{0x1f419ac29aa0?, 0x16dc945?}, {0x1f4199f44168?, 0x0?}}})\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 +0xa7b\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile(0x1f419a212750, {0x3673528?, 0x1f419a212840}, {{{0x1f419ac29aa0, 0x0?}, {0x1f4199f44168?, 0x0?}}})\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 +0x1ab\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler(0x36aaba0, {0x3673560, 0x1f4199f056d0}, {{{0x1f419ac29aa0, 0x2a}, {0x1f4199f44168, 0x14}}}, 0x0)\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 +0x39b\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem(0x36aaba0, {0x3673560, 0x1f4199f056d0})\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 +0x1f8\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1()\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 +0x85\ncreated by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1 in goroutine 109\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:308 +0x26b\n"} k8s.io/apimachinery/pkg/util/runtime.logPanic /go/pkg/mod/k8s.io/apimachinery@v0.36.1/pkg/util/runtime/runtime.go:144 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:201 runtime.gopanic /usr/local/go/src/runtime/panic.go:860 runtime.panicmem /usr/local/go/src/runtime/panic.go:336 runtime.sigpanic /usr/local/go/src/runtime/signal_unix.go:931 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func4 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:223 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.298Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "ec23c905-0761-4fb6-b978-2eaaee6615ba", "error": "panic: runtime error: invalid memory address or nil pointer dereference [recovered]"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312 2026-06-09T07:46:32.298Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "9e5b5ab4-9fbb-4913-b322-c7ddbfb7ca3f"} 2026-06-09T07:46:32.308Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "9e5b5ab4-9fbb-4913-b322-c7ddbfb7ca3f", "state": "error"} 2026-06-09T07:46:32.308Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-s3","namespace":"demand-backup-incremental-gcp-s3-25718"}, "namespace": "demand-backup-incremental-gcp-s3-25718", "name": "backup-gcp-cs-s3", "reconcileID": "9e5b5ab4-9fbb-4913-b322-c7ddbfb7ca3f"} 2026-06-09T07:46:32.308Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "871fb546-14d2-474b-8ac4-b7d80e03270a"} 2026-06-09T07:46:32.319Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "871fb546-14d2-474b-8ac4-b7d80e03270a"} 2026-06-09T07:46:32.319Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "e33627be-c534-4f06-b63b-03e18b843e12"} 2026-06-09T07:46:32.330Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs-native","namespace":"demand-backup-incremental-gcp-native-23181"}, "namespace": "demand-backup-incremental-gcp-native-23181", "name": "backup-gcp-cs-native", "reconcileID": "e33627be-c534-4f06-b63b-03e18b843e12"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-gcp-cs-s3 some-name gcp-cs-s3 error 2m27s Name: backup-gcp-cs-s3 Namespace: demand-backup-incremental-gcp-s3-25718 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-06-09T07:44:10Z Finalizers: percona.com/delete-backup Generation: 1 Resource Version: 1780991192243583019 UID: 734abf36-39fc-4928-91d0-5c48185e1073 Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: gcp-cs-s3 Type: incremental-base Status: Error: starting deadline seconds exceeded State: error Events: Backup object psmdb-backup/backup-gcp-cs-s3 is in error state.