Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/logs/multi-storage.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + cluster=some-name + create_infra multi-storage-13743 + local ns=multi-storage-13743 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.W9hnsooObB ++ mktemp + local LAST_ERR=/tmp/tmp.mmILKjPRZ1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W9hnsooObB customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.mmILKjPRZ1 + rm /tmp/tmp.W9hnsooObB /tmp/tmp.mmILKjPRZ1 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Zo2jtIaz5l ++ mktemp + local LAST_ERR=/tmp/tmp.VSnh009nLp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Zo2jtIaz5l customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.VSnh009nLp + rm /tmp/tmp.Zo2jtIaz5l /tmp/tmp.VSnh009nLp + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.6CIBtSWUSH ++ mktemp + local LAST_ERR=/tmp/tmp.B4vj9L9wwc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6CIBtSWUSH + cat /tmp/tmp.B4vj9L9wwc + rm /tmp/tmp.6CIBtSWUSH /tmp/tmp.B4vj9L9wwc + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.SMMcf90W88 ++ mktemp + local LAST_ERR=/tmp/tmp.gAPCQQE4Wg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SMMcf90W88 + cat /tmp/tmp.gAPCQQE4Wg + rm /tmp/tmp.SMMcf90W88 /tmp/tmp.gAPCQQE4Wg + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.63w2LETpjn ++ mktemp + local LAST_ERR=/tmp/tmp.wPw0STDG2P + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.63w2LETpjn clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.wPw0STDG2P + rm /tmp/tmp.63w2LETpjn /tmp/tmp.wPw0STDG2P + return 0 + check_crd_for_deletion PR-2328-37c432905 + local git_tag=PR-2328-37c432905 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2328-37c432905/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.96PnBsikUf +++ mktemp ++ local LAST_ERR=/tmp/tmp.iRPeyFo780 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.96PnBsikUf ++ cat /tmp/tmp.iRPeyFo780 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.96PnBsikUf ++ cat /tmp/tmp.iRPeyFo780 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.96PnBsikUf ++ cat /tmp/tmp.iRPeyFo780 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.96PnBsikUf ++ cat /tmp/tmp.iRPeyFo780 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.96PnBsikUf /tmp/tmp.iRPeyFo780 ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.9kKPeqgAUb ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.GBozr9MHRf + local LAST_ERR=/tmp/tmp.ahrf0tTgoZ + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.nlSLziEtNd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9kKPeqgAUb + cat /tmp/tmp.ahrf0tTgoZ + rm /tmp/tmp.9kKPeqgAUb /tmp/tmp.ahrf0tTgoZ + return 0 namespace "multi-storage-8851" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GBozr9MHRf namespace "psmdb-operator" deleted + cat /tmp/tmp.nlSLziEtNd + rm /tmp/tmp.GBozr9MHRf /tmp/tmp.nlSLziEtNd + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ABBENvjwG7 ++ mktemp + local LAST_ERR=/tmp/tmp.GG4xQlXblJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ABBENvjwG7 + cat /tmp/tmp.GG4xQlXblJ + rm /tmp/tmp.ABBENvjwG7 /tmp/tmp.GG4xQlXblJ + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.YOjwYrnklf ++ mktemp + local LAST_ERR=/tmp/tmp.4dRklJJuip + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YOjwYrnklf namespace/psmdb-operator created + cat /tmp/tmp.4dRklJJuip + rm /tmp/tmp.YOjwYrnklf /tmp/tmp.4dRklJJuip + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.VLygDyRGOW +++ mktemp ++ local LAST_ERR=/tmp/tmp.4sHOTMTHvy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VLygDyRGOW ++ cat /tmp/tmp.4sHOTMTHvy ++ rm /tmp/tmp.VLygDyRGOW /tmp/tmp.4sHOTMTHvy ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.z6sDFtn5gG ++ mktemp + local LAST_ERR=/tmp/tmp.Bwze4husdx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.z6sDFtn5gG Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1" modified. + cat /tmp/tmp.Bwze4husdx + rm /tmp/tmp.z6sDFtn5gG /tmp/tmp.Bwze4husdx + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/multi-storage/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.dqoX1H0Ua6 ++ mktemp + local LAST_ERR=/tmp/tmp.M6DeJPxYiX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dqoX1H0Ua6 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.M6DeJPxYiX + rm /tmp/tmp.dqoX1H0Ua6 /tmp/tmp.M6DeJPxYiX + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.DsLAvktAqo ++ mktemp + local LAST_ERR=/tmp/tmp.gJx1cbXbIe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DsLAvktAqo clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.gJx1cbXbIe + rm /tmp/tmp.DsLAvktAqo /tmp/tmp.gJx1cbXbIe + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.qzZsZcbS9S ++ mktemp + local LAST_ERR=/tmp/tmp.Cb6c6knvx3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qzZsZcbS9S deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.Cb6c6knvx3 + rm /tmp/tmp.qzZsZcbS9S /tmp/tmp.Cb6c6knvx3 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.f5Aaun4ckp +++ mktemp ++ local LAST_ERR=/tmp/tmp.fdDB3Pk08K ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f5Aaun4ckp ++ cat /tmp/tmp.fdDB3Pk08K ++ rm /tmp/tmp.f5Aaun4ckp /tmp/tmp.fdDB3Pk08K ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7dbf5fc47b-w4w5j + local pod=percona-server-mongodb-operator-7dbf5fc47b-w4w5j + set +o xtrace waiting for pod/percona-server-mongodb-operator-7dbf5fc47b-w4w5j to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.bE0XQgFKZT +++ mktemp ++ local LAST_ERR=/tmp/tmp.edls281pHU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bE0XQgFKZT ++ cat /tmp/tmp.edls281pHU ++ rm /tmp/tmp.bE0XQgFKZT /tmp/tmp.edls281pHU ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-w4w5j ++ mktemp + local LAST_OUT=/tmp/tmp.L3znJ0G8dw ++ mktemp + local LAST_ERR=/tmp/tmp.iBx4hy45ql + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-w4w5j + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L3znJ0G8dw + cat /tmp/tmp.iBx4hy45ql + rm /tmp/tmp.L3znJ0G8dw /tmp/tmp.iBx4hy45ql + return 0 2026-05-21T08:39:46.921Z INFO setup Manager starting up {"gitCommit": "37c432905eccde48178453d4542cff5f669efa6b", "gitBranch": "PR-2328-37c432905", "buildTime": "", "goVersion": "go1.26.3", "os": "linux", "arch": "amd64"} + create_namespace multi-storage-13743 + local namespace=multi-storage-13743 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces multi-storage-13743' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces multi-storage-13743 ----------------------------------------------------------------------------------- + xargs kubectl delete ns + kubectl_bin delete namespace multi-storage-13743 --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.eYYEnaq19y ++ mktemp + local LAST_OUT=/tmp/tmp.OqDDA32xvX ++ mktemp + local LAST_ERR=/tmp/tmp.BF9kVDPpin + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.nog4kp29te + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace multi-storage-13743 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eYYEnaq19y + cat /tmp/tmp.BF9kVDPpin + rm /tmp/tmp.eYYEnaq19y /tmp/tmp.BF9kVDPpin + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OqDDA32xvX + cat /tmp/tmp.nog4kp29te + rm /tmp/tmp.OqDDA32xvX /tmp/tmp.nog4kp29te + return 0 + kubectl_bin wait --for=delete namespace multi-storage-13743 ++ mktemp + local LAST_OUT=/tmp/tmp.p5F3ezTXw9 ++ mktemp + local LAST_ERR=/tmp/tmp.jFsPFsbZfQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace multi-storage-13743 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p5F3ezTXw9 + cat /tmp/tmp.jFsPFsbZfQ + rm /tmp/tmp.p5F3ezTXw9 /tmp/tmp.jFsPFsbZfQ + return 0 + desc 'create namespace multi-storage-13743' + set +o xtrace ----------------------------------------------------------------------------------- create namespace multi-storage-13743 ----------------------------------------------------------------------------------- + kubectl_bin create namespace multi-storage-13743 ++ mktemp + local LAST_OUT=/tmp/tmp.cL7jOf5I9M ++ mktemp + local LAST_ERR=/tmp/tmp.CzI22unVjN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace multi-storage-13743 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cL7jOf5I9M namespace/multi-storage-13743 created + cat /tmp/tmp.CzI22unVjN + rm /tmp/tmp.cL7jOf5I9M /tmp/tmp.CzI22unVjN + return 0 + set_kube_ctx multi-storage-13743 + local namespace=multi-storage-13743 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.zIWseEUMVh +++ mktemp ++ local LAST_ERR=/tmp/tmp.X6ZPFeckLy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zIWseEUMVh ++ cat /tmp/tmp.X6ZPFeckLy ++ rm /tmp/tmp.zIWseEUMVh /tmp/tmp.X6ZPFeckLy ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1 --namespace=multi-storage-13743 ++ mktemp + local LAST_OUT=/tmp/tmp.EhNIe24A0Z ++ mktemp + local LAST_ERR=/tmp/tmp.GPmrmSewVC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1 --namespace=multi-storage-13743 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EhNIe24A0Z Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster1" modified. + cat /tmp/tmp.GPmrmSewVC + rm /tmp/tmp.EhNIe24A0Z /tmp/tmp.GPmrmSewVC + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Thu May 21 08:40:19 2026 NAMESPACE: multi-storage-13743 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.multi-storage-13743.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace multi-storage-13743 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace multi-storage-13743 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace multi-storage-13743 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace multi-storage-13743 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cVew09BG3R +++ mktemp ++ local LAST_ERR=/tmp/tmp.kFo2kHvZJH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cVew09BG3R ++ cat /tmp/tmp.kFo2kHvZJH ++ rm /tmp/tmp.cVew09BG3R /tmp/tmp.kFo2kHvZJH ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-2zpmk + wait_pod minio-service-6d5f646cdc-2zpmk + local pod=minio-service-6d5f646cdc-2zpmk + set +o xtrace waiting for pod/minio-service-6d5f646cdc-2zpmk to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-13743.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.S3P2z0p9f4 ++ mktemp + local LAST_ERR=/tmp/tmp.EQ63oR6cH3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S3P2z0p9f4 make_bucket: operator-testing pod "aws-cli" deleted from multi-storage-13743 namespace + cat /tmp/tmp.EQ63oR6cH3 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-13743 + rm /tmp/tmp.S3P2z0p9f4 /tmp/tmp.EQ63oR6cH3 + return 0 + endpoint=http://minio-service:9000 + create_minio_bucket operator-testing-2 http://minio-service:9000 + local bucket=operator-testing-2 + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2' ++ mktemp + local LAST_OUT=/tmp/tmp.ten3np1G1t ++ mktemp + local LAST_ERR=/tmp/tmp.xrQuOGef7K + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ten3np1G1t make_bucket: operator-testing-2 pod "aws-cli" deleted from multi-storage-13743 namespace + cat /tmp/tmp.xrQuOGef7K All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.ten3np1G1t /tmp/tmp.xrQuOGef7K + return 0 + create_minio_bucket operator-testing-3 http://minio-service:9000 + local bucket=operator-testing-3 + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3' ++ mktemp + local LAST_OUT=/tmp/tmp.JzPWQMpRq3 ++ mktemp + local LAST_ERR=/tmp/tmp.01qzfj4KP1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JzPWQMpRq3 make_bucket: operator-testing-3 make_bucket: operator-testing-3 pod "aws-cli" deleted from multi-storage-13743 namespace + cat /tmp/tmp.01qzfj4KP1 All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-13743 + rm /tmp/tmp.JzPWQMpRq3 /tmp/tmp.01qzfj4KP1 + return 0 + log 'create secrets and start client' + set +o xtrace [2026-05-21T08:41:08+0000] create secrets and start client + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1Oxz3SOSDC ++ mktemp + local LAST_ERR=/tmp/tmp.0doR9kZt7F + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1Oxz3SOSDC secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.0doR9kZt7F + rm /tmp/tmp.1Oxz3SOSDC /tmp/tmp.0doR9kZt7F + return 0 + log 'create PSMDB cluster: some-name' + set +o xtrace [2026-05-21T08:41:10+0000] create PSMDB cluster: some-name + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/multi-storage/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/multi-storage/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/multi-storage/conf/some-name.yml + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + /usr/sbin/sed -e s/NAME_SPACE/multi-storage-13743/g + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905"' + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + local LAST_OUT=/tmp/tmp.RLSZ72xxnK ++ mktemp + local LAST_ERR=/tmp/tmp.Jc0CFildWV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RLSZ72xxnK perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.Jc0CFildWV + rm /tmp/tmp.RLSZ72xxnK /tmp/tmp.Jc0CFildWV + return 0 + log 'check if all 3 Pods started' + set +o xtrace [2026-05-21T08:41:12+0000] check if all 3 Pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KAok19jR1Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.5LtmGv6Lpf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KAok19jR1Z ++ cat /tmp/tmp.5LtmGv6Lpf ++ rm /tmp/tmp.KAok19jR1Z /tmp/tmp.5LtmGv6Lpf ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready..............OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wOWclJfzYU +++ mktemp ++ local LAST_ERR=/tmp/tmp.CaBAMKqXIR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wOWclJfzYU ++ cat /tmp/tmp.CaBAMKqXIR ++ rm /tmp/tmp.wOWclJfzYU /tmp/tmp.CaBAMKqXIR ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uSK7eTyJVc +++ mktemp ++ local LAST_ERR=/tmp/tmp.qml9oTUboH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uSK7eTyJVc ++ cat /tmp/tmp.qml9oTUboH ++ rm /tmp/tmp.uSK7eTyJVc /tmp/tmp.qml9oTUboH ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness......... + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-05-21T08:42:33.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-05-21T08:43:05.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-05-21T08:43:09.000+0000 I listening for the commands + create_user + log 'create user myApp' + set +o xtrace [2026-05-21T08:43:12+0000] create user myApp + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.multi-storage-13743 mongodb + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.multi-storage-13743 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.multi-storage-13743 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jfVauu9MZU +++ mktemp ++ local LAST_ERR=/tmp/tmp.x6axmFzC3g ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jfVauu9MZU ++ cat /tmp/tmp.x6axmFzC3g ++ rm /tmp/tmp.jfVauu9MZU /tmp/tmp.x6axmFzC3g ++ return 0 + local client_container=psmdb-client-bb8b97679-l2sz2 + kubectl_bin exec psmdb-client-bb8b97679-l2sz2 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-13743.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.6IAIUU7Sva ++ mktemp + local LAST_ERR=/tmp/tmp.tILgyZORAP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l2sz2 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-13743.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6IAIUU7Sva Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-13743.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("5b18da06-b3e5-4314-9103-84cb7c165e00") } Percona Server for MongoDB server version: v8.0.21-9 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.tILgyZORAP + rm /tmp/tmp.6IAIUU7Sva /tmp/tmp.tILgyZORAP + return 0 + write_document 100500 + local x=100500 + local cmp_postfix= + log 'write document: 100500' + set +o xtrace [2026-05-21T08:43:15+0000] write document: 100500 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.multi-storage-13743 mongodb + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-13743 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-13743 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rVY9H9yfcX +++ mktemp ++ local LAST_ERR=/tmp/tmp.snrAhX9JgN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rVY9H9yfcX ++ cat /tmp/tmp.snrAhX9JgN ++ rm /tmp/tmp.rVY9H9yfcX /tmp/tmp.snrAhX9JgN ++ return 0 + local client_container=psmdb-client-bb8b97679-l2sz2 + kubectl_bin exec psmdb-client-bb8b97679-l2sz2 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-13743.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.q9LiW9ozAn ++ mktemp + local LAST_ERR=/tmp/tmp.eNuEQhUHCd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-l2sz2 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-13743.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q9LiW9ozAn Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-13743.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1aaafe26-c4cb-4a54-b149-ddf479293e32") } Percona Server for MongoDB server version: v8.0.21-9 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.eNuEQhUHCd + rm /tmp/tmp.q9LiW9ozAn /tmp/tmp.eNuEQhUHCd + return 0 + run_backup minio-1 backup-minio-1 logical + local storage=minio-1 + local name=backup-minio-1 + local type=logical + log 'creating backup: backup-minio-1 (logical)' + set +o xtrace [2026-05-21T08:43:17+0000] creating backup: backup-minio-1 (logical) + yq eval $'.metadata.name = "backup-minio-1"\n\t\t\t| .spec.storageName = "minio-1"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/multi-storage/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9linX3KEAY ++ mktemp + local LAST_ERR=/tmp/tmp.8OtKFBMta1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9linX3KEAY perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.8OtKFBMta1 + rm /tmp/tmp.9linX3KEAY /tmp/tmp.8OtKFBMta1 + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state.2026-05-21T08:39:46.921Z INFO setup Manager starting up {"gitCommit": "37c432905eccde48178453d4542cff5f669efa6b", "gitBranch": "PR-2328-37c432905", "buildTime": "", "goVersion": "go1.26.3", "os": "linux", "arch": "amd64"} 2026-05-21T08:39:47.165Z INFO setup Multi-cluster services (MCS) are not available: ServiceExport resource not found in cluster 2026-05-21T08:39:47.180Z INFO server version {"platform": "kubernetes", "version": "v1.33.11-gke.1197000"} 2026-05-21T08:39:47.209Z INFO controller-runtime.metrics Starting metrics server 2026-05-21T08:39:47.209Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-05-21T08:39:47.209Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} I0521 08:39:47.209619 1 leaderelection.go:258] "Attempting to acquire leader lease..." lock="psmdb-operator/08db0feb.percona.com" I0521 08:39:47.242711 1 leaderelection.go:272] "Successfully acquired lease" lock="psmdb-operator/08db0feb.percona.com" 2026-05-21T08:39:47.242Z DEBUG events percona-server-mongodb-operator-7dbf5fc47b-w4w5j_b0ad3690-1288-44a0-ac46-59bfcbf0f494 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"psmdb-operator","name":"08db0feb.percona.com","uid":"bb11de70-6c13-406f-abbf-e084e2309233","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1779352787234991015"}, "reason": "LeaderElection"} 2026-05-21T08:39:47.244Z INFO Starting EventSource {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.Pod"} 2026-05-21T08:39:47.244Z INFO Starting EventSource {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "source": "kind source: *v1.PerconaServerMongoDB"} 2026-05-21T08:39:47.244Z INFO Starting EventSource {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.PerconaServerMongoDBBackup"} 2026-05-21T08:39:47.244Z INFO Starting EventSource {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.Pod"} 2026-05-21T08:39:47.244Z INFO Starting EventSource {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.PerconaServerMongoDBRestore"} 2026-05-21T08:39:47.757Z INFO Starting Controller {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB"} 2026-05-21T08:39:47.757Z INFO Starting workers {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "worker count": 1} 2026-05-21T08:39:47.758Z INFO Starting Controller {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore"} 2026-05-21T08:39:47.758Z INFO Starting Controller {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup"} 2026-05-21T08:39:47.758Z INFO Starting workers {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "worker count": 1} 2026-05-21T08:39:47.758Z INFO Starting workers {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "worker count": 1} 2026-05-21T08:41:12.807Z INFO Set CR version {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "version": "1.23.0"} 2026-05-21T08:41:14.388Z INFO Created a new mongo key {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "KeyName": "some-name-mongodb-keyfile"} 2026-05-21T08:41:14.421Z INFO Created a new mongo key {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "KeyName": "some-name-mongodb-encryption-key"} 2026-05-21T08:41:14.481Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "name": "some-name-rs0", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}", "generation": 0, "resourceVersion": ""} 2026-05-21T08:41:14.851Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "name": "some-name-rs0", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "generation": 0, "resourceVersion": ""} 2026-05-21T08:41:14.889Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "name": "some-name-mongod-rs0", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "generation": 0, "resourceVersion": ""} 2026-05-21T08:41:15.139Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:15.139Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:15.448Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:15.448Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "dbefc16b-8337-4bc1-8ede-8c4f47520b75", "previous": "", "current": "initializing"} 2026-05-21T08:41:16.589Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "7835cdb5-3a5c-4e63-8b11-f2da62c36ce2", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:16.589Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "7835cdb5-3a5c-4e63-8b11-f2da62c36ce2", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:16.907Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "7835cdb5-3a5c-4e63-8b11-f2da62c36ce2", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:17.987Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "4ac30867-ed02-4347-be8c-cc05e23cd5d2", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:17.987Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "4ac30867-ed02-4347-be8c-cc05e23cd5d2", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:18.297Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "4ac30867-ed02-4347-be8c-cc05e23cd5d2", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:24.333Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "40a36acd-5b6c-41e3-9d3f-cc9dd656b371", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:24.333Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "40a36acd-5b6c-41e3-9d3f-cc9dd656b371", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:24.646Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "40a36acd-5b6c-41e3-9d3f-cc9dd656b371", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:30.749Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "729c15c4-6629-400d-b168-b38e66b9cce2", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:30.749Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "729c15c4-6629-400d-b168-b38e66b9cce2", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:31.035Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "729c15c4-6629-400d-b168-b38e66b9cce2", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:37.108Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "c5c8409f-8209-4b5e-a06a-51e770cfedb0", "replset": "rs0", "size": 3, "pods": 1} 2026-05-21T08:41:37.108Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "c5c8409f-8209-4b5e-a06a-51e770cfedb0", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:37.429Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "c5c8409f-8209-4b5e-a06a-51e770cfedb0", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:44.351Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "44ae712a-f94a-4c4f-97eb-b604e111a330", "replset": "rs0", "size": 3, "pods": 2} 2026-05-21T08:41:44.351Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "44ae712a-f94a-4c4f-97eb-b604e111a330", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:44.701Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "44ae712a-f94a-4c4f-97eb-b604e111a330", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:45.781Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "f48f722b-b14b-49f7-becd-0f9b167437ef", "replset": "rs0", "size": 3, "pods": 2} 2026-05-21T08:41:45.781Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "f48f722b-b14b-49f7-becd-0f9b167437ef", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:46.460Z INFO PBM pbm-agent version {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "f48f722b-b14b-49f7-becd-0f9b167437ef", "pod": "some-name-rs0-0", "image": "docker.io/perconalab/percona-server-mongodb-operator:main-backup", "version": "2.14.0"} 2026-05-21T08:41:46.480Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "f48f722b-b14b-49f7-becd-0f9b167437ef", "replset": "rs0"} 2026-05-21T08:41:46.644Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "f48f722b-b14b-49f7-becd-0f9b167437ef", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:47.758Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "6d7e76aa-d1dc-4a09-948e-a0395a3bc3da", "replset": "rs0", "size": 3, "pods": 2} 2026-05-21T08:41:47.758Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "6d7e76aa-d1dc-4a09-948e-a0395a3bc3da", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:48.035Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "6d7e76aa-d1dc-4a09-948e-a0395a3bc3da", "replset": "rs0"} 2026-05-21T08:41:48.210Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "6d7e76aa-d1dc-4a09-948e-a0395a3bc3da", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:41:54.321Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "cb51fc09-9c2e-46d6-8ede-ad4105ddb237", "replset": "rs0", "size": 3, "pods": 2} 2026-05-21T08:41:54.321Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "cb51fc09-9c2e-46d6-8ede-ad4105ddb237", "replset": "rs0", "status": "initializing"} 2026-05-21T08:41:54.574Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "cb51fc09-9c2e-46d6-8ede-ad4105ddb237", "replset": "rs0"} 2026-05-21T08:41:54.750Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "cb51fc09-9c2e-46d6-8ede-ad4105ddb237", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:42:00.974Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1d29153d-af49-4fea-9b4b-26a016c1530b", "replset": "rs0", "size": 3, "pods": 2} 2026-05-21T08:42:00.974Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1d29153d-af49-4fea-9b4b-26a016c1530b", "replset": "rs0", "status": "initializing"} 2026-05-21T08:42:01.385Z DEBUG PBM waiting for replset to be initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1d29153d-af49-4fea-9b4b-26a016c1530b", "replset": "rs0"} 2026-05-21T08:42:01.550Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1d29153d-af49-4fea-9b4b-26a016c1530b", "pbmStatus": "initializing", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2026-05-21T08:42:18.165Z INFO initiating replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0", "pod": "some-name-rs0-0"} 2026-05-21T08:42:25.441Z INFO some-name-rs0-0 is the writable primary {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0"} 2026-05-21T08:42:25.441Z INFO replset initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0", "pod": "some-name-rs0-0"} 2026-05-21T08:42:25.441Z INFO creating user admin {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2026-05-21T08:42:28.550Z INFO user admin created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2026-05-21T08:42:28.944Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "database": "admin", "user": "clusterAdmin"} 2026-05-21T08:42:29.254Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "database": "admin", "user": "clusterMonitor"} 2026-05-21T08:42:29.369Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "database": "admin", "user": "backup"} 2026-05-21T08:42:29.447Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "database": "admin", "user": "databaseAdmin"} 2026-05-21T08:42:29.494Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "replset": "rs0", "status": "initializing"} 2026-05-21T08:42:29.899Z INFO PBM configuration changed or resync is needed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "oldHash": "", "newHash": "017ffaa04988df80592649f447f8bf268b3d7398f0c60f9c87cf348d0559a587"} 2026-05-21T08:42:29.918Z INFO PBM Setting config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "cluster": "some-name", "mainStorage": "minio-1"} 2026-05-21T08:42:29.957Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "name": "minio-2", "type": "s3"} 2026-05-21T08:42:29.978Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "name": "minio-3", "type": "s3"} 2026-05-21T08:42:30.007Z INFO PBM waiting for resync to start (up to 30 seconds) {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03"} 2026-05-21T08:43:00.059Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "64a2f67e-6962-4133-a8c3-84025b469f03", "error": "reconcile PBM: reconcile configuration: resync: wait for resync: wait for resync to start: resync is not started until deadline", "errorVerbose": "resync is not started until deadline\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1229\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1244\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nwait for resync to start\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).WaitForResync\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1245\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nwait for resync\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup.(*pbmC).ResyncMainStorageAndWait\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/pbm.go:1210\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:181\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nresync\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBMConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:182\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:42\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile configuration\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcilePBM\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/pbm.go:43\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:516\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile PBM\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:518\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-05-21T08:43:01.695Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "_id": 1, "host": "some-name-rs0-1.some-name-rs0.multi-storage-13743.svc.cluster.local:27017"} 2026-05-21T08:43:01.695Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "replset": "rs0"} 2026-05-21T08:43:01.695Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "cfg": {"_id":"rs0","version":2,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-z4gd","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-4j4d","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a0ec56e1b6583150f89238b"},"writeConcernMajorityJournalDefault":true}} 2026-05-21T08:43:01.713Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "replset": "rs0"} 2026-05-21T08:43:01.713Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "cfg": {"_id":"rs0","version":3,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-z4gd","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-4j4d","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":0}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a0ec56e1b6583150f89238b"},"writeConcernMajorityJournalDefault":true}} 2026-05-21T08:43:01.722Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "liveMembers": 1, "pods": 3} 2026-05-21T08:43:01.724Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "replset": "rs0", "status": "initializing"} 2026-05-21T08:43:02.236Z INFO PBM configuration changed or resync is needed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "oldHash": "", "newHash": "017ffaa04988df80592649f447f8bf268b3d7398f0c60f9c87cf348d0559a587"} 2026-05-21T08:43:02.263Z INFO PBM Setting config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "cluster": "some-name", "mainStorage": "minio-1"} 2026-05-21T08:43:02.285Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "name": "minio-2", "type": "s3"} 2026-05-21T08:43:02.302Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "name": "minio-3", "type": "s3"} 2026-05-21T08:43:02.669Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"} 2026-05-21T08:43:02.669Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "1f958117-13f7-45d8-8ea9-72a59ccfe14a", "previous": "error", "current": "initializing"} 2026-05-21T08:43:04.129Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "_id": 2, "host": "some-name-rs0-2.some-name-rs0.multi-storage-13743.svc.cluster.local:27017"} 2026-05-21T08:43:04.129Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "replset": "rs0"} 2026-05-21T08:43:04.129Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "cfg": {"_id":"rs0","version":5,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-z4gd","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-4j4d","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":0},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-89h5","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a0ec56e1b6583150f89238b"},"writeConcernMajorityJournalDefault":true}} 2026-05-21T08:43:04.150Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "replset": "rs0"} 2026-05-21T08:43:04.150Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "cfg": {"_id":"rs0","version":6,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-z4gd","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-4j4d","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.multi-storage-13743.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2328-37c43-default-pool-e7443ad8-89h5","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6a0ec56e1b6583150f89238b"},"writeConcernMajorityJournalDefault":true}} 2026-05-21T08:43:04.164Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "liveMembers": 2, "pods": 3} 2026-05-21T08:43:04.166Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "replset": "rs0", "status": "initializing"} 2026-05-21T08:43:04.761Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8291541d-7ce9-4d66-92a8-9dccf31df3e7", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"} 2026-05-21T08:43:07.244Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "36be0b82-6b22-4b1d-983a-d7bbc9358bd0", "previous": "initializing", "current": "ready"} 2026-05-21T08:43:08.191Z INFO update Mongo version to 8.0.21-9 (fetched from db) {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "ef3c7de2-8b1d-41d1-99e1-76abd21b1401"} 2026-05-21T08:43:09.258Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio-1). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "ef3c7de2-8b1d-41d1-99e1-76abd21b1401"} 2026-05-21T08:43:11.712Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio-1). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "8ed973c2-f0ba-44fa-82f2-0c8039800272"} 2026-05-21T08:43:19.357Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687"} 2026-05-21T08:43:19.372Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "state": ""} 2026-05-21T08:43:19.620Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "cluster": "some-name", "namespace": "multi-storage-13743"} 2026-05-21T08:43:19.620Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-05-21T08:43:19.673Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687"} 2026-05-21T08:43:19.817Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "finalizer": "internal.percona.com/release-lock"} 2026-05-21T08:43:19.817Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "backup": "backup-minio-1", "storage": "minio-1"} 2026-05-21T08:43:19.817Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "backup": "backup-minio-1", "storage": "minio-1", "backupCmd": "backup [name: 2026-05-21T08:43:19Z, compression: gzip (level: default)] ", "profile": ""} 2026-05-21T08:43:19.826Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687", "previous": "", "current": "requested"} 2026-05-21T08:43:19.857Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "ce86a9e2-4032-43bb-9b07-0ee7f1ced687"} 2026-05-21T08:43:19.857Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c"} 2026-05-21T08:43:19.873Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "state": "requested"} 2026-05-21T08:43:19.923Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "cluster": "some-name", "namespace": "multi-storage-13743"} 2026-05-21T08:43:19.923Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "currentJob": {"Name":"backup-minio-1","Type":0}} 2026-05-21T08:43:20.124Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c"} 2026-05-21T08:43:20.153Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "backup": "backup-minio-1", "error": "acquire backup lock: create lease: leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists", "errorVerbose": "leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists\ncreate lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.AcquireLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:39\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:276\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nacquire backup lock\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:278\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:148 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-05-21T08:43:20.153Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "previous": "requested", "current": "error"} 2026-05-21T08:43:20.184Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "lease": "psmdb-some-name-backup-lock"} 2026-05-21T08:43:20.184Z ERROR failed to release the lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "error": "get lease: Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found", "errorVerbose": "Lease.coordination.k8s.io \"psmdb-some-name-backup-lock\" not found\nget lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.ReleaseLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:66\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:162\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:164 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:243 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-05-21T08:43:20.184Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c"} 2026-05-21T08:43:20.184Z INFO Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c"} 2026-05-21T08:43:20.184Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "1ca6a46f-8c43-4167-af9e-1e2f41a5e42c", "error": "reconcile backup: acquire backup lock: create lease: leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists", "errorVerbose": "leases.coordination.k8s.io \"psmdb-some-name-backup-lock\" already exists\ncreate lease\ngithub.com/percona/percona-server-mongodb-operator/pkg/k8s.AcquireLease\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/k8s/lease.go:39\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:276\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nacquire backup lock\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:278\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:239\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:495 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313 2026-05-21T08:43:20.184Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "39402833-b2a2-4958-b801-678014be2fde"} 2026-05-21T08:43:20.197Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "39402833-b2a2-4958-b801-678014be2fde", "state": "error"} 2026-05-21T08:43:20.197Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "39402833-b2a2-4958-b801-678014be2fde"} 2026-05-21T08:43:20.343Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "9dd3fffa-8734-453b-acc4-a695dc3a2e94"} 2026-05-21T08:43:20.356Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "9dd3fffa-8734-453b-acc4-a695dc3a2e94", "state": "error"} 2026-05-21T08:43:20.356Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "backup-minio-1", "reconcileID": "9dd3fffa-8734-453b-acc4-a695dc3a2e94"} 2026-05-21T08:43:21.363Z INFO PBM Point-in-time recovery will work only with full backup in main storage (minio-1). Please create one manually or wait for scheduled backup to be created (if configured). {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"multi-storage-13743"}, "namespace": "multi-storage-13743", "name": "some-name", "reconcileID": "bfeade44-c81f-4b8a-b26c-545e430c06ef"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-minio-1 some-name minio-1 s3://operator-testing/2026-05-21T08:43:19Z error 5s Name: backup-minio-1 Namespace: multi-storage-13743 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2026-05-21T08:43:19Z Finalizers: percona.com/delete-backup internal.percona.com/release-lock Generation: 1 Resource Version: 1779353000176719021 UID: 6f1f5d5d-9998-40aa-81c8-f438dc8a04d0 Spec: Cluster Name: some-name Starting Deadline Seconds: 120 Storage Name: minio-1 Type: logical Status: Destination: s3://operator-testing/2026-05-21T08:43:19Z Error: acquire backup lock: create lease: leases.coordination.k8s.io "psmdb-some-name-backup-lock" already exists Last Transition: 2026-05-21T08:43:19Z Pbm Name: 2026-05-21T08:43:19Z Replset Names: rs0 s3: Bucket: operator-testing Credentials Secret: minio-secret Endpoint URL: http://minio-service:9000/ Region: us-east-1 Server Side Encryption: State: error Storage Name: minio-1 Events: Backup object psmdb-backup/backup-minio-1 is in error state.