Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/logs/demand-backup-physical-minio-native-tls.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + create_infra demand-backup-physical-minio-native-tls-9869 + local ns=demand-backup-physical-minio-native-tls-9869 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.cvxwSCThEB ++ mktemp + local LAST_ERR=/tmp/tmp.GkkcFl1UXh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cvxwSCThEB customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.GkkcFl1UXh + rm /tmp/tmp.cvxwSCThEB /tmp/tmp.GkkcFl1UXh + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.eZa5z6AEt1 ++ mktemp + local LAST_ERR=/tmp/tmp.0mjOs65jmY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eZa5z6AEt1 + cat /tmp/tmp.0mjOs65jmY + rm /tmp/tmp.eZa5z6AEt1 /tmp/tmp.0mjOs65jmY + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.CE01sEh3zc ++ mktemp + local LAST_ERR=/tmp/tmp.H85nbjaGQN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CE01sEh3zc + cat /tmp/tmp.H85nbjaGQN + rm /tmp/tmp.CE01sEh3zc /tmp/tmp.H85nbjaGQN + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.EmqtNsbQNF ++ mktemp + local LAST_ERR=/tmp/tmp.d7L6phl4ul + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EmqtNsbQNF customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met + cat /tmp/tmp.d7L6phl4ul + rm /tmp/tmp.EmqtNsbQNF /tmp/tmp.d7L6phl4ul + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.7VMvb6EjdA ++ mktemp + local LAST_ERR=/tmp/tmp.iHSAod9Jbz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7VMvb6EjdA clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.iHSAod9Jbz + rm /tmp/tmp.7VMvb6EjdA /tmp/tmp.iHSAod9Jbz + return 0 + check_crd_for_deletion PR-2328-37c432905 + local git_tag=PR-2328-37c432905 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2328-37c432905/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LTIRpycRo8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vRzfOyk9Ev ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.LTIRpycRo8 ++ cat /tmp/tmp.vRzfOyk9Ev Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.LTIRpycRo8 ++ cat /tmp/tmp.vRzfOyk9Ev Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.LTIRpycRo8 ++ cat /tmp/tmp.vRzfOyk9Ev Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.LTIRpycRo8 ++ cat /tmp/tmp.vRzfOyk9Ev Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.LTIRpycRo8 /tmp/tmp.vRzfOyk9Ev ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' ++ mktemp + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.124YActdrb ++ mktemp + local LAST_OUT=/tmp/tmp.6boK9YThnQ ++ mktemp + local LAST_ERR=/tmp/tmp.cBl1ACiR09 + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.iCeSGyQWta + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.124YActdrb + cat /tmp/tmp.cBl1ACiR09 + rm /tmp/tmp.124YActdrb /tmp/tmp.cBl1ACiR09 + return 0 namespace "cert-manager" deleted namespace "demand-backup-physical-minio-native-tls-4402" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6boK9YThnQ namespace "psmdb-operator" deleted + cat /tmp/tmp.iCeSGyQWta + rm /tmp/tmp.6boK9YThnQ /tmp/tmp.iCeSGyQWta + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.BfpHixf9zq ++ mktemp + local LAST_ERR=/tmp/tmp.VfZlVz5apg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BfpHixf9zq + cat /tmp/tmp.VfZlVz5apg + rm /tmp/tmp.BfpHixf9zq /tmp/tmp.VfZlVz5apg + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.CkySRROEnU ++ mktemp + local LAST_ERR=/tmp/tmp.WXq1ZXjW9J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CkySRROEnU namespace/psmdb-operator created + cat /tmp/tmp.WXq1ZXjW9J + rm /tmp/tmp.CkySRROEnU /tmp/tmp.WXq1ZXjW9J + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.cammpmIVKa +++ mktemp ++ local LAST_ERR=/tmp/tmp.ConlTv9WK6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cammpmIVKa ++ cat /tmp/tmp.ConlTv9WK6 ++ rm /tmp/tmp.cammpmIVKa /tmp/tmp.ConlTv9WK6 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.dxakdB5D46 ++ mktemp + local LAST_ERR=/tmp/tmp.49jzjmDw5V + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dxakdB5D46 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12" modified. + cat /tmp/tmp.49jzjmDw5V + rm /tmp/tmp.dxakdB5D46 /tmp/tmp.49jzjmDw5V + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.dGDh3QW8NG ++ mktemp + local LAST_ERR=/tmp/tmp.zErWA5wZGE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dGDh3QW8NG customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.zErWA5wZGE + rm /tmp/tmp.dGDh3QW8NG /tmp/tmp.zErWA5wZGE + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.rURVIqAKTa ++ mktemp + local LAST_ERR=/tmp/tmp.Yl7gSqXWgm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rURVIqAKTa clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.Yl7gSqXWgm + rm /tmp/tmp.rURVIqAKTa /tmp/tmp.Yl7gSqXWgm + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.H9jlZNUXXn ++ mktemp + local LAST_ERR=/tmp/tmp.H3norW5djW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H9jlZNUXXn deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.H3norW5djW + rm /tmp/tmp.H9jlZNUXXn /tmp/tmp.H3norW5djW + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.lC59FQR7I5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xKHzD4Czhl ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lC59FQR7I5 ++ cat /tmp/tmp.xKHzD4Czhl ++ rm /tmp/tmp.lC59FQR7I5 /tmp/tmp.xKHzD4Czhl ++ return 0 + wait_operator_pod percona-server-mongodb-operator-7dbf5fc47b-tf2xz + local pod=percona-server-mongodb-operator-7dbf5fc47b-tf2xz + set +o xtrace waiting for pod/percona-server-mongodb-operator-7dbf5fc47b-tf2xz to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.tbSgpA3j6d +++ mktemp ++ local LAST_ERR=/tmp/tmp.BH6SgNY5xz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tbSgpA3j6d ++ cat /tmp/tmp.BH6SgNY5xz ++ rm /tmp/tmp.tbSgpA3j6d /tmp/tmp.BH6SgNY5xz ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-tf2xz ++ mktemp + local LAST_OUT=/tmp/tmp.ZhoI55Q3cs ++ mktemp + local LAST_ERR=/tmp/tmp.BzUDkDB9kB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-tf2xz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZhoI55Q3cs + cat /tmp/tmp.BzUDkDB9kB + rm /tmp/tmp.ZhoI55Q3cs /tmp/tmp.BzUDkDB9kB + return 0 2026-05-21T08:06:55.787Z INFO setup Manager starting up {"gitCommit": "37c432905eccde48178453d4542cff5f669efa6b", "gitBranch": "PR-2328-37c432905", "buildTime": "", "goVersion": "go1.26.3", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-minio-native-tls-9869 + local namespace=demand-backup-physical-minio-native-tls-9869 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-physical-minio-native-tls-9869' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-minio-native-tls-9869 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-minio-native-tls-9869 --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.7PHU4XtQCv ++ mktemp + local LAST_OUT=/tmp/tmp.Urp0bXhyTO ++ mktemp + local LAST_ERR=/tmp/tmp.K5Yzh0l9AN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.0l7HdLTrkK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-physical-minio-native-tls-9869 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7PHU4XtQCv + cat /tmp/tmp.K5Yzh0l9AN + rm /tmp/tmp.7PHU4XtQCv /tmp/tmp.K5Yzh0l9AN + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Urp0bXhyTO + cat /tmp/tmp.0l7HdLTrkK + rm /tmp/tmp.Urp0bXhyTO /tmp/tmp.0l7HdLTrkK + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-minio-native-tls-9869 ++ mktemp + local LAST_OUT=/tmp/tmp.DqiUVe2qwU ++ mktemp + local LAST_ERR=/tmp/tmp.OCewhAIVgw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-physical-minio-native-tls-9869 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DqiUVe2qwU + cat /tmp/tmp.OCewhAIVgw + rm /tmp/tmp.DqiUVe2qwU /tmp/tmp.OCewhAIVgw + return 0 + desc 'create namespace demand-backup-physical-minio-native-tls-9869' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-minio-native-tls-9869 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-minio-native-tls-9869 ++ mktemp + local LAST_OUT=/tmp/tmp.fGlpAsAaO4 ++ mktemp + local LAST_ERR=/tmp/tmp.ur5QruJCtr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-physical-minio-native-tls-9869 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fGlpAsAaO4 namespace/demand-backup-physical-minio-native-tls-9869 created + cat /tmp/tmp.ur5QruJCtr + rm /tmp/tmp.fGlpAsAaO4 /tmp/tmp.ur5QruJCtr + return 0 + set_kube_ctx demand-backup-physical-minio-native-tls-9869 + local namespace=demand-backup-physical-minio-native-tls-9869 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.daxjgJSZrv +++ mktemp ++ local LAST_ERR=/tmp/tmp.MZiQfVD1kP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.daxjgJSZrv ++ cat /tmp/tmp.MZiQfVD1kP ++ rm /tmp/tmp.daxjgJSZrv /tmp/tmp.MZiQfVD1kP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12 --namespace=demand-backup-physical-minio-native-tls-9869 ++ mktemp + local LAST_OUT=/tmp/tmp.Q1ZVQv8DoJ ++ mktemp + local LAST_ERR=/tmp/tmp.TIJBiQ29Ff + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12 --namespace=demand-backup-physical-minio-native-tls-9869 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Q1ZVQv8DoJ Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2328-37c432905-3-cluster12" modified. + cat /tmp/tmp.TIJBiQ29Ff + rm /tmp/tmp.Q1ZVQv8DoJ /tmp/tmp.TIJBiQ29Ff + return 0 + desc 'Deploy cert-manager for proper certificate generation' + set +o xtrace ----------------------------------------------------------------------------------- Deploy cert-manager for proper certificate generation ----------------------------------------------------------------------------------- + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.AhNEEhxrzQ ++ mktemp + local LAST_ERR=/tmp/tmp.RxmdKC7P19 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AhNEEhxrzQ namespace/cert-manager created + cat /tmp/tmp.RxmdKC7P19 + rm /tmp/tmp.AhNEEhxrzQ /tmp/tmp.RxmdKC7P19 + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.bvPOiJ5owm ++ mktemp + local LAST_ERR=/tmp/tmp.zqitxaMUwk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bvPOiJ5owm namespace/cert-manager labeled + cat /tmp/tmp.zqitxaMUwk + rm /tmp/tmp.bvPOiJ5owm /tmp/tmp.zqitxaMUwk + return 0 + kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.t44OvMseNH ++ mktemp + local LAST_ERR=/tmp/tmp.68xTD0o8zd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t44OvMseNH namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.68xTD0o8zd Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.t44OvMseNH /tmp/tmp.68xTD0o8zd + return 0 + kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready ++ mktemp + local LAST_OUT=/tmp/tmp.E2dj2XH0Ry ++ mktemp + local LAST_ERR=/tmp/tmp.t3OH7hvk56 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E2dj2XH0Ry pod/cert-manager-559d798845-jm7cq condition met pod/cert-manager-cainjector-64958d9c7c-7fr8w condition met pod/cert-manager-webhook-7fb6f99b56-rjb95 condition met + cat /tmp/tmp.t3OH7hvk56 E0521 08:07:45.648582 550548 reflector.go:227] "Failed to watch" err="Get \"https://34.70.203.56/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-559d798845-jm7cq&resourceVersion=1779350865169550000&timeoutSeconds=307&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" E0521 08:07:45.950407 550548 reflector.go:227] "Failed to watch" err="Get \"https://34.70.203.56/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-7fr8w&resourceVersion=1779350865169550000&timeoutSeconds=432&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.E2dj2XH0Ry /tmp/tmp.t3OH7hvk56 + return 0 + sleep 120 + desc 'Create self-signed issuer' + set +o xtrace ----------------------------------------------------------------------------------- Create self-signed issuer ----------------------------------------------------------------------------------- + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.eYc3zg7u4b ++ mktemp + local LAST_ERR=/tmp/tmp.JXQu2qhA40 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eYc3zg7u4b issuer.cert-manager.io/selfsigned-issuer created + cat /tmp/tmp.JXQu2qhA40 + rm /tmp/tmp.eYc3zg7u4b /tmp/tmp.JXQu2qhA40 + return 0 + desc 'Generate certificate for first MinIO via cert-manager' + set +o xtrace ----------------------------------------------------------------------------------- Generate certificate for first MinIO via cert-manager ----------------------------------------------------------------------------------- + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.av4FqlpEMn ++ mktemp + local LAST_ERR=/tmp/tmp.8IF4FE2Pud + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.av4FqlpEMn certificate.cert-manager.io/minio-cert created + cat /tmp/tmp.8IF4FE2Pud Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. + rm /tmp/tmp.av4FqlpEMn /tmp/tmp.8IF4FE2Pud + return 0 + kubectl_bin wait --for=condition=Ready certificate/minio-cert -n demand-backup-physical-minio-native-tls-9869 --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.hCLm1xwI3Z ++ mktemp + local LAST_ERR=/tmp/tmp.ZmwpLNZ781 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready certificate/minio-cert -n demand-backup-physical-minio-native-tls-9869 --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hCLm1xwI3Z certificate.cert-manager.io/minio-cert condition met + cat /tmp/tmp.ZmwpLNZ781 E0521 08:09:56.319752 564576 reflector.go:227] "Failed to watch" err="Get \"https://34.70.203.56/apis/cert-manager.io/v1/namespaces/demand-backup-physical-minio-native-tls-9869/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminio-cert&resourceVersion=1779350995773887010&timeoutSeconds=436&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.hCLm1xwI3Z /tmp/tmp.ZmwpLNZ781 + return 0 + kubectl_bin get secret minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.crt}' + base64 -d ++ mktemp + local LAST_OUT=/tmp/tmp.q896RflzDD ++ mktemp + local LAST_ERR=/tmp/tmp.ZjgUUViKf1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.crt}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q896RflzDD + cat /tmp/tmp.ZjgUUViKf1 + rm /tmp/tmp.q896RflzDD /tmp/tmp.ZjgUUViKf1 + return 0 + kubectl_bin get secret minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.key}' + base64 -d ++ mktemp + local LAST_OUT=/tmp/tmp.mXEg8iI4RP ++ mktemp + local LAST_ERR=/tmp/tmp.09Ot36yKYZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.key}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mXEg8iI4RP + cat /tmp/tmp.09Ot36yKYZ + rm /tmp/tmp.mXEg8iI4RP /tmp/tmp.09Ot36yKYZ + return 0 + kubectl_bin create secret generic minio-tls -n demand-backup-physical-minio-native-tls-9869 --from-file=public.crt=/tmp/minio-cert.pem --from-file=private.key=/tmp/minio-key.pem ++ mktemp + local LAST_OUT=/tmp/tmp.sewbmNCe2N ++ mktemp + local LAST_ERR=/tmp/tmp.clNPCD97tZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic minio-tls -n demand-backup-physical-minio-native-tls-9869 --from-file=public.crt=/tmp/minio-cert.pem --from-file=private.key=/tmp/minio-key.pem + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sewbmNCe2N secret/minio-tls created + cat /tmp/tmp.clNPCD97tZ + rm /tmp/tmp.sewbmNCe2N /tmp/tmp.clNPCD97tZ + return 0 + kubectl_bin create secret generic minio-ca-bundle -n demand-backup-physical-minio-native-tls-9869 --from-file=ca.crt=/tmp/minio-cert.pem ++ mktemp + local LAST_OUT=/tmp/tmp.gC3orMM4UJ ++ mktemp + local LAST_ERR=/tmp/tmp.kI7nTZ7THI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic minio-ca-bundle -n demand-backup-physical-minio-native-tls-9869 --from-file=ca.crt=/tmp/minio-cert.pem + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gC3orMM4UJ secret/minio-ca-bundle created + cat /tmp/tmp.kI7nTZ7THI + rm /tmp/tmp.gC3orMM4UJ /tmp/tmp.kI7nTZ7THI + return 0 + desc 'Deploy first MinIO with TLS' + set +o xtrace ----------------------------------------------------------------------------------- Deploy first MinIO with TLS ----------------------------------------------------------------------------------- + deploy_minio minio-tls + local cert_secret=minio-tls + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n minio-tls ]] + endpoint=https://minio-service:9000 + minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret") + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa --set tls.enabled=true --set tls.certSecret=minio-tls minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa --set tls.enabled=true --set tls.certSecret=minio-tls minio/minio NAME: minio-service LAST DEPLOYED: Thu May 21 08:10:02 2026 NAMESPACE: demand-backup-physical-minio-native-tls-9869 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-physical-minio-native-tls-9869.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-minio-native-tls-9869 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-minio-native-tls-9869 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-minio-native-tls-9869 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-minio-native-tls-9869 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ayiUocvl6Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.pyd7h0Df1X ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ayiUocvl6Q ++ cat /tmp/tmp.pyd7h0Df1X ++ rm /tmp/tmp.ayiUocvl6Q /tmp/tmp.pyd7h0Df1X ++ return 0 + local MINIO_POD=minio-service-6f4667bf4c-pq2rb + wait_pod minio-service-6f4667bf4c-pq2rb + local pod=minio-service-6f4667bf4c-pq2rb + set +o xtrace waiting for pod/minio-service-6f4667bf4c-pq2rb to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-minio-native-tls-9869.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing https://minio-service:9000 + local bucket=operator-testing + local endpoint=https://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.rDWUxcnTVn ++ mktemp + local LAST_ERR=/tmp/tmp.VKWwYEeZgc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rDWUxcnTVn /usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html InsecureRequestWarning) make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-physical-minio-native-tls-9869 namespace + cat /tmp/tmp.VKWwYEeZgc All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.rDWUxcnTVn /tmp/tmp.VKWwYEeZgc + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.HI5M3k9YJY ++ mktemp + local LAST_ERR=/tmp/tmp.a9UEvvcP2U + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HI5M3k9YJY secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.a9UEvvcP2U + rm /tmp/tmp.HI5M3k9YJY /tmp/tmp.a9UEvvcP2U + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/secondary-minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.QHxAKKjzr6 ++ mktemp + local LAST_ERR=/tmp/tmp.OB9qcaXaA8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/secondary-minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QHxAKKjzr6 secret/secondary-minio-secret created + cat /tmp/tmp.OB9qcaXaA8 + rm /tmp/tmp.QHxAKKjzr6 /tmp/tmp.OB9qcaXaA8 + return 0 + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.l9Yn2OA6zM ++ mktemp + local LAST_ERR=/tmp/tmp.3EEmTn9SZ3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l9Yn2OA6zM secret/some-users created + cat /tmp/tmp.3EEmTn9SZ3 + rm /tmp/tmp.l9Yn2OA6zM /tmp/tmp.3EEmTn9SZ3 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UNYjhwJJIH ++ mktemp + local LAST_ERR=/tmp/tmp.mCj9fbcnnS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UNYjhwJJIH deployment.apps/psmdb-client created + cat /tmp/tmp.mCj9fbcnnS + rm /tmp/tmp.UNYjhwJJIH /tmp/tmp.mCj9fbcnnS + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/some-name-single-ca.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/some-name-single-ca.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/some-name-single-ca.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2328-37c432905"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-physical-minio-native-tls-9869/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.9S7Y0RXAhW ++ mktemp + local LAST_ERR=/tmp/tmp.EA5ougeGk1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9S7Y0RXAhW perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.EA5ougeGk1 + rm /tmp/tmp.9S7Y0RXAhW /tmp/tmp.EA5ougeGk1 + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R6E5BzS2cp +++ mktemp ++ local LAST_ERR=/tmp/tmp.MiqUHJJCFm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.R6E5BzS2cp ++ cat /tmp/tmp.MiqUHJJCFm ++ rm /tmp/tmp.R6E5BzS2cp /tmp/tmp.MiqUHJJCFm ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zmSdmiA1nn +++ mktemp ++ local LAST_ERR=/tmp/tmp.zD6dwpIWiS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zmSdmiA1nn ++ cat /tmp/tmp.zD6dwpIWiS ++ rm /tmp/tmp.zmSdmiA1nn /tmp/tmp.zD6dwpIWiS ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WODfMK9pJG +++ mktemp ++ local LAST_ERR=/tmp/tmp.0WfAjnx4W6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WODfMK9pJG ++ cat /tmp/tmp.0WfAjnx4W6 ++ rm /tmp/tmp.WODfMK9pJG /tmp/tmp.0WfAjnx4W6 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jISooZD1g1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WFFjRSM0Cj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jISooZD1g1 ++ cat /tmp/tmp.WFFjRSM0Cj ++ rm /tmp/tmp.jISooZD1g1 /tmp/tmp.WFFjRSM0Cj ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'Verify CA bundle infrastructure' + set +o xtrace ----------------------------------------------------------------------------------- Verify CA bundle infrastructure ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- test -f /etc/s3/certs/ca-bundle.crt ++ mktemp + local LAST_OUT=/tmp/tmp.3c3E1nmdgN ++ mktemp + local LAST_ERR=/tmp/tmp.cWqPl1F6Ie + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- test -f /etc/s3/certs/ca-bundle.crt + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3c3E1nmdgN + cat /tmp/tmp.cWqPl1F6Ie + rm /tmp/tmp.3c3E1nmdgN /tmp/tmp.cWqPl1F6Ie + return 0 + kubectl_bin exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- env + grep SSL_CERT_FILE ++ mktemp + local LAST_OUT=/tmp/tmp.7slqRTYyxC ++ mktemp + local LAST_ERR=/tmp/tmp.LbQtxaHyqr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- env + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7slqRTYyxC + cat /tmp/tmp.LbQtxaHyqr + rm /tmp/tmp.7slqRTYyxC /tmp/tmp.LbQtxaHyqr + return 0 SSL_CERT_FILE=/etc/s3/certs/ca-bundle.crt ++ kubectl_bin exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- grep -c 'BEGIN CERTIFICATE' /etc/s3/certs/ca-bundle.crt +++ mktemp ++ local LAST_OUT=/tmp/tmp.DAGfkyPufe +++ mktemp ++ local LAST_ERR=/tmp/tmp.47U7qzDIad ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- grep -c 'BEGIN CERTIFICATE' /etc/s3/certs/ca-bundle.crt ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DAGfkyPufe ++ cat /tmp/tmp.47U7qzDIad ++ rm /tmp/tmp.DAGfkyPufe /tmp/tmp.47U7qzDIad ++ return 0 + cert_count=1 + '[' 1 = 1 ']' + desc 'Generate certificate for secondary MinIO' + set +o xtrace ----------------------------------------------------------------------------------- Generate certificate for secondary MinIO ----------------------------------------------------------------------------------- + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.KzAPpKy04u ++ mktemp + local LAST_ERR=/tmp/tmp.hLAC8bufQZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KzAPpKy04u certificate.cert-manager.io/secondary-minio-cert created + cat /tmp/tmp.hLAC8bufQZ Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`. + rm /tmp/tmp.KzAPpKy04u /tmp/tmp.hLAC8bufQZ + return 0 + kubectl_bin wait --for=condition=Ready certificate/secondary-minio-cert -n demand-backup-physical-minio-native-tls-9869 --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.1cA8TI4aOh ++ mktemp + local LAST_ERR=/tmp/tmp.CSnDFplQ9u + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready certificate/secondary-minio-cert -n demand-backup-physical-minio-native-tls-9869 --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1cA8TI4aOh certificate.cert-manager.io/secondary-minio-cert condition met + cat /tmp/tmp.CSnDFplQ9u E0521 08:12:35.674259 585224 reflector.go:227] "Failed to watch" err="Get \"https://34.70.203.56/apis/cert-manager.io/v1/namespaces/demand-backup-physical-minio-native-tls-9869/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsecondary-minio-cert&resourceVersion=1779351154867455015&timeoutSeconds=437&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.1cA8TI4aOh /tmp/tmp.CSnDFplQ9u + return 0 + kubectl_bin get secret secondary-minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.crt}' + base64 -d ++ mktemp + local LAST_OUT=/tmp/tmp.VcX8VxMN28 ++ mktemp + local LAST_ERR=/tmp/tmp.dvPvxvytHk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret secondary-minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.crt}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VcX8VxMN28 + cat /tmp/tmp.dvPvxvytHk + rm /tmp/tmp.VcX8VxMN28 /tmp/tmp.dvPvxvytHk + return 0 + kubectl_bin get secret secondary-minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.key}' + base64 -d ++ mktemp + local LAST_OUT=/tmp/tmp.riqtIJsEeq ++ mktemp + local LAST_ERR=/tmp/tmp.yJwDiU1vLs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get secret secondary-minio-tls-temp -n demand-backup-physical-minio-native-tls-9869 -o 'jsonpath={.data.tls\.key}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.riqtIJsEeq + cat /tmp/tmp.yJwDiU1vLs + rm /tmp/tmp.riqtIJsEeq /tmp/tmp.yJwDiU1vLs + return 0 + kubectl_bin create secret generic secondary-minio-tls -n demand-backup-physical-minio-native-tls-9869 --from-file=public.crt=/tmp/secondary-minio-cert.pem --from-file=private.key=/tmp/secondary-minio-key.pem ++ mktemp + local LAST_OUT=/tmp/tmp.eQ7BY4XXF3 ++ mktemp + local LAST_ERR=/tmp/tmp.l9lAZzJQaJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic secondary-minio-tls -n demand-backup-physical-minio-native-tls-9869 --from-file=public.crt=/tmp/secondary-minio-cert.pem --from-file=private.key=/tmp/secondary-minio-key.pem + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eQ7BY4XXF3 secret/secondary-minio-tls created + cat /tmp/tmp.l9lAZzJQaJ + rm /tmp/tmp.eQ7BY4XXF3 /tmp/tmp.l9lAZzJQaJ + return 0 + kubectl_bin create secret generic secondary-minio-ca-bundle -n demand-backup-physical-minio-native-tls-9869 --from-file=ca.crt=/tmp/secondary-minio-cert.pem ++ mktemp + local LAST_OUT=/tmp/tmp.62KzoDxtoZ ++ mktemp + local LAST_ERR=/tmp/tmp.8UgTMdsDKk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create secret generic secondary-minio-ca-bundle -n demand-backup-physical-minio-native-tls-9869 --from-file=ca.crt=/tmp/secondary-minio-cert.pem + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.62KzoDxtoZ secret/secondary-minio-ca-bundle created + cat /tmp/tmp.8UgTMdsDKk + rm /tmp/tmp.62KzoDxtoZ /tmp/tmp.8UgTMdsDKk + return 0 + deploy_minio secondary-minio-tls secondary-minio-service + local cert_secret=secondary-minio-tls + local service_name=secondary-minio-service + desc 'install MinIO: secondary-minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: secondary-minio-service ----------------------------------------------------------------------------------- + helm uninstall secondary-minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://secondary-minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=secondary-minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=secondary-minio-service-sa') + local minio_args + [[ -n secondary-minio-tls ]] + endpoint=https://secondary-minio-service:9000 + minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret") + retry 10 60 helm install secondary-minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=secondary-minio-service --set serviceAccount.create=true --set serviceAccount.name=secondary-minio-service-sa --set tls.enabled=true --set tls.certSecret=secondary-minio-tls minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install secondary-minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=secondary-minio-service --set serviceAccount.create=true --set serviceAccount.name=secondary-minio-service-sa --set tls.enabled=true --set tls.certSecret=secondary-minio-tls minio/minio NAME: secondary-minio-service LAST DEPLOYED: Thu May 21 08:12:42 2026 NAMESPACE: demand-backup-physical-minio-native-tls-9869 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: secondary-minio-service.demand-backup-physical-minio-native-tls-9869.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-minio-native-tls-9869 -l "release=secondary-minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-minio-native-tls-9869 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_secondary-minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-minio-native-tls-9869 secondary-minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-minio-native-tls-9869 secondary-minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls secondary-minio-service-local ++ kubectl_bin get pods --selector=release=secondary-minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nGJFIvWZaF +++ mktemp ++ local LAST_ERR=/tmp/tmp.zzFLsn8tJ6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=secondary-minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nGJFIvWZaF ++ cat /tmp/tmp.zzFLsn8tJ6 ++ rm /tmp/tmp.nGJFIvWZaF /tmp/tmp.zzFLsn8tJ6 ++ return 0 + local MINIO_POD=secondary-minio-service-656846748c-kh2fq + wait_pod secondary-minio-service-656846748c-kh2fq + local pod=secondary-minio-service-656846748c-kh2fq + set +o xtrace waiting for pod/secondary-minio-service-656846748c-kh2fq to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname secondary-minio-service --external-name=secondary-minio-service.demand-backup-physical-minio-native-tls-9869.svc.cluster.local --tcp=9000 service/secondary-minio-service created + create_minio_bucket operator-testing https://secondary-minio-service:9000 + local bucket=operator-testing + local endpoint=https://secondary-minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://secondary-minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.otFLfkF3CF ++ mktemp + local LAST_ERR=/tmp/tmp.lALiHVt9LV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://secondary-minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.otFLfkF3CF /usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html InsecureRequestWarning) make_bucket: operator-testing /usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html InsecureRequestWarning) make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-physical-minio-native-tls-9869 namespace + cat /tmp/tmp.lALiHVt9LV All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-physical-minio-native-tls-9869 + rm /tmp/tmp.otFLfkF3CF /tmp/tmp.lALiHVt9LV + return 0 + desc 'Add second storage (secondary)' + set +o xtrace ----------------------------------------------------------------------------------- Add second storage (secondary) ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name -n demand-backup-physical-minio-native-tls-9869 --type=merge -p $'\n{\n "spec": {\n "backup": {\n "storages": {\n "secondary": {\n "type": "minio",\n "minio": {\n "credentialsSecret": "secondary-minio-secret",\n "region": "us-east-1",\n "bucket": "operator-testing",\n "endpointUrl": "https://secondary-minio-service:9000",\n "secure": true,\n "insecureSkipTLSVerify": false,\n "caBundle": {\n "name": "secondary-minio-ca-bundle",\n "key": "ca.crt"\n }\n }\n }\n }\n }\n }\n}' ++ mktemp + local LAST_OUT=/tmp/tmp.DKlXRWQ42E ++ mktemp + local LAST_ERR=/tmp/tmp.PP34piYpqg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name -n demand-backup-physical-minio-native-tls-9869 --type=merge -p $'\n{\n "spec": {\n "backup": {\n "storages": {\n "secondary": {\n "type": "minio",\n "minio": {\n "credentialsSecret": "secondary-minio-secret",\n "region": "us-east-1",\n "bucket": "operator-testing",\n "endpointUrl": "https://secondary-minio-service:9000",\n "secure": true,\n "insecureSkipTLSVerify": false,\n "caBundle": {\n "name": "secondary-minio-ca-bundle",\n "key": "ca.crt"\n }\n }\n }\n }\n }\n }\n}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DKlXRWQ42E perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.PP34piYpqg + rm /tmp/tmp.DKlXRWQ42E /tmp/tmp.PP34piYpqg + return 0 + sleep 300 + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p4hMGxfhWN +++ mktemp ++ local LAST_ERR=/tmp/tmp.tCCmxu4JUq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.p4hMGxfhWN ++ cat /tmp/tmp.tCCmxu4JUq ++ rm /tmp/tmp.p4hMGxfhWN /tmp/tmp.tCCmxu4JUq ++ return 0 + [[ ready == ready ]] + echo .OK .OK + desc 'Verify merged CA bundle' + set +o xtrace ----------------------------------------------------------------------------------- Verify merged CA bundle ----------------------------------------------------------------------------------- ++ kubectl_bin exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- grep -c 'BEGIN CERTIFICATE' /etc/s3/certs/ca-bundle.crt +++ mktemp ++ local LAST_OUT=/tmp/tmp.GFAdMpGsT2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zaPXZpkfkT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec some-name-rs0-0 -n demand-backup-physical-minio-native-tls-9869 -c backup-agent -- grep -c 'BEGIN CERTIFICATE' /etc/s3/certs/ca-bundle.crt ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GFAdMpGsT2 ++ cat /tmp/tmp.zaPXZpkfkT ++ rm /tmp/tmp.GFAdMpGsT2 /tmp/tmp.zaPXZpkfkT ++ return 0 + cert_count=2 + '[' 2 = 2 ']' + desc 'Backup & restore with merged CA' + set +o xtrace ----------------------------------------------------------------------------------- Backup & restore with merged CA ----------------------------------------------------------------------------------- + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + run_mongo_tls 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yBSC6zG9Cf +++ mktemp ++ local LAST_ERR=/tmp/tmp.34RCJasTUy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yBSC6zG9Cf ++ cat /tmp/tmp.34RCJasTUy ++ rm /tmp/tmp.yBSC6zG9Cf /tmp/tmp.34RCJasTUy ++ return 0 + local client_container=psmdb-client-5649fbb65f-npvr6 + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-minio-native-tls-9869 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.tSFWOvHrP0 ++ mktemp + local LAST_ERR=/tmp/tmp.nWilPZpUMo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tSFWOvHrP0 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-05-21T08:18:33.044Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("caa51754-44a3-478b-86a5-cb1cc3156f89") } Percona Server for MongoDB server version: v8.0.21-9 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.nWilPZpUMo + rm /tmp/tmp.tSFWOvHrP0 /tmp/tmp.nWilPZpUMo + return 0 + run_mongo_tls 'db.getSiblingDB("myApp").test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local 'command=db.getSiblingDB("myApp").test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hjoVvJ0qQY +++ mktemp ++ local LAST_ERR=/tmp/tmp.kyOpeqGvB1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hjoVvJ0qQY ++ cat /tmp/tmp.kyOpeqGvB1 ++ rm /tmp/tmp.hjoVvJ0qQY /tmp/tmp.kyOpeqGvB1 ++ return 0 + local client_container=psmdb-client-5649fbb65f-npvr6 + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.getSiblingDB("myApp").test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.wun9bGAPWg ++ mktemp + local LAST_ERR=/tmp/tmp.Ohd2S1FiIR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.getSiblingDB("myApp").test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wun9bGAPWg Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-05-21T08:18:35.571Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("85808b43-a90c-407b-900f-07843d82c469") } Percona Server for MongoDB server version: v8.0.21-9 WARNING: shell and server versions do not match WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Ohd2S1FiIR + rm /tmp/tmp.wun9bGAPWg /tmp/tmp.Ohd2S1FiIR + return 0 + backup_name_single=backup-minio + run_backup minio backup-minio physical + local storage=minio + local backup_name=backup-minio + local type=physical + log 'running backup backup-minio' + set +o xtrace [2026-05-21T08:18:35+0000] running backup backup-minio + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZJy54vv8Aj ++ mktemp + local LAST_ERR=/tmp/tmp.TaM2t5R5zT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZJy54vv8Aj perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.TaM2t5R5zT + rm /tmp/tmp.ZJy54vv8Aj /tmp/tmp.TaM2t5R5zT + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state.......OK + check_backup_in_storage backup-minio minio rs0 filelist.pbm https + local backup=backup-minio + local storage_type=minio + local replset=rs0 + local file=filelist.pbm + local protocol=https + local endpoint ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|azure://||' ++ local LAST_OUT=/tmp/tmp.abVz9BBQYk +++ mktemp ++ local LAST_ERR=/tmp/tmp.YkDWcTmp61 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.abVz9BBQYk ++ cat /tmp/tmp.YkDWcTmp61 ++ rm /tmp/tmp.abVz9BBQYk /tmp/tmp.YkDWcTmp61 ++ return 0 + backup_dest=operator-testing/2026-05-21T08:18:38Z + case ${storage_type} in + endpoint=minio-service + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://minio-service:9000 s3 ls s3://operator-testing/2026-05-21T08:18:38Z/rs0/filelist.pbm + grep filelist.pbm ++ mktemp + local LAST_OUT=/tmp/tmp.I1N9VUE2ih ++ mktemp + local LAST_ERR=/tmp/tmp.XI9dEgX3LA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url https://minio-service:9000 s3 ls s3://operator-testing/2026-05-21T08:18:38Z/rs0/filelist.pbm + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I1N9VUE2ih + cat /tmp/tmp.XI9dEgX3LA All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-physical-minio-native-tls-9869 + rm /tmp/tmp.I1N9VUE2ih /tmp/tmp.XI9dEgX3LA + return 0 2026-05-21 08:18:45 15792 filelist.pbm 2026-05-21 08:18:45 15792 filelist.pbm + run_mongo_tls 'db.getSiblingDB("myApp").test.drop()' myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local 'command=db.getSiblingDB("myApp").test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VJAUlDERxy +++ mktemp ++ local LAST_ERR=/tmp/tmp.0hwiyl8dGo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VJAUlDERxy ++ cat /tmp/tmp.0hwiyl8dGo ++ rm /tmp/tmp.VJAUlDERxy /tmp/tmp.0hwiyl8dGo ++ return 0 + local client_container=psmdb-client-5649fbb65f-npvr6 + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.getSiblingDB("myApp").test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.yRCCRvKZT9 ++ mktemp + local LAST_ERR=/tmp/tmp.vU6Fu2Xpkj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-npvr6 -- bash -c 'printf '\''db.getSiblingDB("myApp").test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yRCCRvKZT9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-05-21T08:18:57.332Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("930ef1bb-e76b-4535-b22a-526d40831eeb") } Percona Server for MongoDB server version: v8.0.21-9 WARNING: shell and server versions do not match true bye + cat /tmp/tmp.vU6Fu2Xpkj + rm /tmp/tmp.yRCCRvKZT9 /tmp/tmp.vU6Fu2Xpkj + return 0 + run_restore backup-minio + local backup_name=backup-minio + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/restore.yml + log 'running restore restore-backup-minio' + set +o xtrace [2026-05-21T08:18:57+0000] running restore restore-backup-minio + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tfbU1ai7jh ++ mktemp + local LAST_ERR=/tmp/tmp.O5W6vlhjRJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tfbU1ai7jh perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created + cat /tmp/tmp.O5W6vlhjRJ + rm /tmp/tmp.tfbU1ai7jh /tmp/tmp.O5W6vlhjRJ + return 0 + run_recovery_check backup-minio '' '' true + local backup_name=backup-minio + local compare_suffix=_restore + local is_sharded= + local tls=true + wait_restore backup-minio some-name requested 0 3000 + local backup_name=backup-minio + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio object to be created.OK Waiting psmdb-restore/restore-backup-minio to reach state "requested" ..OK after 1 minutes + [[ 0 -eq 1 ]] + echo + compare_kubectl statefulset/some-name-rs0 _restore + local resource=statefulset/some-name-rs0 + local postfix=_restore + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/compare/statefulset_some-name-rs0_restore.yml + local new_result=/tmp/tmp.I2W1X8LGox/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/compare/statefulset_some-name-rs0_restore-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-physical-minio-native-tls-9869", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.TKEG5xHkUE ++ mktemp + local LAST_ERR=/tmp/tmp.vSyfOTVG6N + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TKEG5xHkUE + cat /tmp/tmp.vSyfOTVG6N + rm /tmp/tmp.TKEG5xHkUE /tmp/tmp.vSyfOTVG6N + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.I2W1X8LGox/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.I2W1X8LGox/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.I2W1X8LGox/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/compare/statefulset_some-name-rs0_restore.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2328/e2e-tests/demand-backup-physical-minio-native-tls/compare/statefulset_some-name-rs0_restore.yml /tmp/tmp.I2W1X8LGox/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-05-21T08:20:39+0000] compare_kubectl: statefulset/some-name-rs0 OK + wait_restore backup-minio some-name ready 0 3000 + local backup_name=backup-minio + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio object to be created.OK Waiting psmdb-restore/restore-backup-minio to reach state "ready" ..... ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.H5hZBTJtPS ++ mktemp + local LAST_ERR=/tmp/tmp.LpAntd3t8r + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H5hZBTJtPS 2026/05/21 08:20:07 [entrypoint] starting `pbm-agent` 2026-05-21T08:20:11.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-05-21T08:20:11.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-05-21T08:20:11.000+0000 I starting PITR routine 2026-05-21T08:20:11.000+0000 I pbm-agent: Version: 2.14.0 Platform: linux/amd64 GitCommit: d3436ef3d07ae8de58d6d0dd942d5084eb85e3a6 GitBranch: release-2.14.0 BuildTime: 2026-04-24_15:23_UTC GoVersion: go1.25.9 2026-05-21T08:20:11.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017 2026-05-21T08:20:11.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-05-21T08:20:11.000+0000 I listening for the commands 2026-05-21T08:20:38.000+0000 I got command restore [name: 2026-05-21T08:20:38.394170544Z, snapshot: 2026-05-21T08:18:38Z] , opid: 6a0ec0567a795b9658b504fc 2026-05-21T08:20:38.000+0000 I got epoch {1779351539 2} 2026-05-21T08:20:38.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] waiting for PITR config disabled and OpLock released 2026-05-21T08:20:43.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] PITR slicer stopped 2026-05-21T08:20:43.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] backup: 2026-05-21T08:18:38Z 2026-05-21T08:20:43.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] recovery started 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] port: 28064 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] download max buf 536870912 (arena 268435456, span 33554432, concurrency 2) 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] mongod binary: mongod, version: v8.0.21-9 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] restore opts: fallbackEnabled: false; allowPartlyDone: true 2026-05-21T08:20:43.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] moving to state starting 2026-05-21T08:20:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.starting" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:43.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] waiting for cluster 2026-05-21T08:22:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:22:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:22:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:43.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:53.000+0000 E [restore/2026-05-21T08:20:38.394170544Z] mark error during restore: move to running state: wait for cluster: cluster failed: 1779351889:move to running state: wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644 2026-05-21T08:24:53.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.error" [size hint: 333 (333.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:53.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] set error on cluster level 2026-05-21T08:24:53.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.error" [size hint: 333 (333.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:53.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] wait for cluster status 2026-05-21T08:24:58.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.cleanup.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:58.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] no cleanup strategy to apply 2026-05-21T08:24:58.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] recovery successfully finished 2026-05-21T08:24:58.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] cleanup heartbeats stopped 2026-05-21T08:24:58.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] heartbeats stopped + cat /tmp/tmp.LpAntd3t8r Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.H5hZBTJtPS /tmp/tmp.LpAntd3t8r + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.Yw6OcJs5S5 ++ mktemp + local LAST_ERR=/tmp/tmp.MqHOW6YGB2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Yw6OcJs5S5 2026/05/21 08:20:49 [entrypoint] starting `pbm-agent` 2026-05-21T08:20:53.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-05-21T08:20:53.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-05-21T08:20:53.000+0000 I pbm-agent: Version: 2.14.0 Platform: linux/amd64 GitCommit: d3436ef3d07ae8de58d6d0dd942d5084eb85e3a6 GitBranch: release-2.14.0 BuildTime: 2026-04-24_15:23_UTC GoVersion: go1.25.9 2026-05-21T08:20:53.000+0000 I starting PITR routine 2026-05-21T08:20:53.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017 2026-05-21T08:20:53.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-05-21T08:20:53.000+0000 I listening for the commands + cat /tmp/tmp.MqHOW6YGB2 Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.Yw6OcJs5S5 /tmp/tmp.MqHOW6YGB2 + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.1XpUmUUwjD ++ mktemp + local LAST_ERR=/tmp/tmp.nQeMNktlSP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1XpUmUUwjD 2026/05/21 08:19:34 [entrypoint] starting `pbm-agent` 2026-05-21T08:19:39.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-05-21T08:19:39.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-05-21T08:19:39.000+0000 I pbm-agent: Version: 2.14.0 Platform: linux/amd64 GitCommit: d3436ef3d07ae8de58d6d0dd942d5084eb85e3a6 GitBranch: release-2.14.0 BuildTime: 2026-04-24_15:23_UTC GoVersion: go1.25.9 2026-05-21T08:19:39.000+0000 I starting PITR routine 2026-05-21T08:19:39.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017 2026-05-21T08:19:39.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-05-21T08:19:39.000+0000 I listening for the commands 2026-05-21T08:20:39.000+0000 I got command restore [name: 2026-05-21T08:20:38.394170544Z, snapshot: 2026-05-21T08:18:38Z] , opid: 6a0ec0567a795b9658b504fc 2026-05-21T08:20:39.000+0000 I got epoch {1779351539 2} 2026-05-21T08:20:39.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] oplog slicer disabled 2026-05-21T08:20:39.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] waiting for PITR config disabled and OpLock released 2026-05-21T08:20:44.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] PITR slicer stopped 2026-05-21T08:20:44.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] backup: 2026-05-21T08:18:38Z 2026-05-21T08:20:44.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] recovery started 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] port: 27494 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] download max buf 536870912 (arena 268435456, span 33554432, concurrency 2) 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] mongod binary: mongod, version: v8.0.21-9 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] restore opts: fallbackEnabled: false; allowPartlyDone: true 2026-05-21T08:20:44.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] moving to state starting 2026-05-21T08:20:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.starting" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:20:44.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] waiting for `starting` status in rs map[.pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017:{} .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017:{} .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017:{}] 2026-05-21T08:22:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:22:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:22:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:44.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/rs.error" [size hint: 242 (242.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.error" [size hint: 242 (242.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:49.000+0000 E [restore/2026-05-21T08:20:38.394170544Z] mark error during restore: move to running state: wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-2.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.error" [size hint: 265 (265.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] set error on cluster level 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.error" [size hint: 265 (265.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:49.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] wait for cluster status 2026-05-21T08:24:54.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] uploading ".pbm.restore/2026-05-21T08:20:38.394170544Z/cluster.cleanup.hb" [size hint: 10 (10.00B); part size: 10485760 (10.00MB)] 2026-05-21T08:24:54.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] no cleanup strategy to apply 2026-05-21T08:24:54.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] cleanup heartbeats stopped 2026-05-21T08:24:54.000+0000 D [restore/2026-05-21T08:20:38.394170544Z] heartbeats stopped 2026-05-21T08:24:54.000+0000 I [restore/2026-05-21T08:20:38.394170544Z] recovery successfully finished + cat /tmp/tmp.nQeMNktlSP Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.1XpUmUUwjD /tmp/tmp.nQeMNktlSP + return 0 + grep restore-backup-minio + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.ClPr3wAOzj +++ mktemp ++ local LAST_ERR=/tmp/tmp.2tduWkgyN1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ClPr3wAOzj ++ cat /tmp/tmp.2tduWkgyN1 ++ rm /tmp/tmp.ClPr3wAOzj /tmp/tmp.2tduWkgyN1 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-tf2xz ++ mktemp + local LAST_OUT=/tmp/tmp.jiqKBfpbJg ++ mktemp + local LAST_ERR=/tmp/tmp.Z7lXPg1pZj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-7dbf5fc47b-tf2xz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jiqKBfpbJg + cat /tmp/tmp.Z7lXPg1pZj + rm /tmp/tmp.jiqKBfpbJg /tmp/tmp.Z7lXPg1pZj + return 0 2026-05-21T08:23:36.707Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "fae6de66-5dce-49f2-8ac4-623deec2a233", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:23:41.708Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:23:41.709Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "pod": "some-name-rs0-1"} 2026-05-21T08:23:41.709Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "pod": "some-name-rs0-2"} 2026-05-21T08:23:41.709Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "pod": "some-name-rs0-0"} 2026-05-21T08:23:41.709Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:23:41.709Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:23:42.138Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "65536c84-d39b-4e7a-a945-6459220ced4b", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:23:47.139Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:23:47.140Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "pod": "some-name-rs0-0"} 2026-05-21T08:23:47.140Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "pod": "some-name-rs0-1"} 2026-05-21T08:23:47.140Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "pod": "some-name-rs0-2"} 2026-05-21T08:23:47.140Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:23:47.140Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:23:47.639Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "23c7a388-6cfe-4a4f-aaf2-fcda5877dd92", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:23:52.640Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:23:52.641Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "pod": "some-name-rs0-0"} 2026-05-21T08:23:52.641Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "pod": "some-name-rs0-1"} 2026-05-21T08:23:52.641Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "pod": "some-name-rs0-2"} 2026-05-21T08:23:52.641Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:23:52.641Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:23:53.133Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4fbb6483-49ce-4650-a97e-d297121f3643", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:23:58.135Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:23:58.136Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "pod": "some-name-rs0-0"} 2026-05-21T08:23:58.136Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "pod": "some-name-rs0-1"} 2026-05-21T08:23:58.136Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "pod": "some-name-rs0-2"} 2026-05-21T08:23:58.136Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:23:58.136Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:23:58.535Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "3909ff48-cf62-4001-9608-234971f71205", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:03.536Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:03.536Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "pod": "some-name-rs0-0"} 2026-05-21T08:24:03.536Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "pod": "some-name-rs0-1"} 2026-05-21T08:24:03.536Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "pod": "some-name-rs0-2"} 2026-05-21T08:24:03.536Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:03.536Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:04.020Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "e3517aa8-9951-447a-9c35-eacf938c5b99", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:09.021Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:09.022Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "pod": "some-name-rs0-0"} 2026-05-21T08:24:09.022Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "pod": "some-name-rs0-1"} 2026-05-21T08:24:09.022Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "pod": "some-name-rs0-2"} 2026-05-21T08:24:09.022Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:09.022Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:09.435Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0bd9dc82-2076-4071-8344-a2a4cec53e3a", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:14.436Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:14.437Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "pod": "some-name-rs0-0"} 2026-05-21T08:24:14.437Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "pod": "some-name-rs0-1"} 2026-05-21T08:24:14.437Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "pod": "some-name-rs0-2"} 2026-05-21T08:24:14.437Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:14.437Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:14.841Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8c58d8db-f55a-4e72-9b5b-03b57d610377", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:19.842Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:19.843Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "pod": "some-name-rs0-0"} 2026-05-21T08:24:19.843Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "pod": "some-name-rs0-1"} 2026-05-21T08:24:19.843Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "pod": "some-name-rs0-2"} 2026-05-21T08:24:19.843Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:19.843Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:20.085Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20064d06-e946-437e-a895-7c07d4368284", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:25.086Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:25.087Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "pod": "some-name-rs0-0"} 2026-05-21T08:24:25.088Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "pod": "some-name-rs0-1"} 2026-05-21T08:24:25.088Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "pod": "some-name-rs0-2"} 2026-05-21T08:24:25.088Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:25.088Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:25.327Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "97c3c69c-2464-4dad-a0b7-6642385aee86", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:30.328Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:30.329Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "pod": "some-name-rs0-0"} 2026-05-21T08:24:30.329Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "pod": "some-name-rs0-1"} 2026-05-21T08:24:30.329Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "pod": "some-name-rs0-2"} 2026-05-21T08:24:30.329Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:30.329Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:30.953Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "4900af65-3489-4074-9100-953d3171fda7", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:35.954Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:35.955Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "pod": "some-name-rs0-2"} 2026-05-21T08:24:35.955Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "pod": "some-name-rs0-0"} 2026-05-21T08:24:35.955Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "pod": "some-name-rs0-1"} 2026-05-21T08:24:35.955Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:35.955Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:36.405Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "1971e9b4-b550-48ec-9f38-bf87967efc54", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:41.407Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:41.407Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "pod": "some-name-rs0-0"} 2026-05-21T08:24:41.407Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "pod": "some-name-rs0-1"} 2026-05-21T08:24:41.407Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "pod": "some-name-rs0-2"} 2026-05-21T08:24:41.407Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:41.408Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:41.831Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "8f559efa-7885-4f9a-91d4-ca46a481a50c", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:46.832Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:46.832Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "pod": "some-name-rs0-0"} 2026-05-21T08:24:46.833Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "pod": "some-name-rs0-1"} 2026-05-21T08:24:46.833Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "pod": "some-name-rs0-2"} 2026-05-21T08:24:46.833Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:46.833Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:47.231Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "20e1ad37-abb4-4a4a-ab5e-590cda07fb09", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-05-21T08:24:52.232Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "cluster": "some-name", "namespace": "demand-backup-physical-minio-native-tls-9869"} 2026-05-21T08:24:52.233Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "pod": "some-name-rs0-2"} 2026-05-21T08:24:52.233Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "pod": "some-name-rs0-0"} 2026-05-21T08:24:52.234Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "pod": "some-name-rs0-1"} 2026-05-21T08:24:52.234Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "sts": "some-name-rs0", "replset": "rs0"} 2026-05-21T08:24:52.234Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "command": ["/opt/percona/pbm", "describe-restore", "2026-05-21T08:20:38.394170544Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-05-21T08:24:52.727Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "status": {"type":"physical","opid":"","name":"2026-05-21T08:20:38.394170544Z","replsets":[{"name":"rs0","start_ts":0,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":1779351889,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","error":"wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":1779351889,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"error","conditions":null,"n":null,"error":"move to running state: wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644","pbm_version":"","balancer":""}} 2026-05-21T08:24:52.727Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"}, "namespace": "demand-backup-physical-minio-native-tls-9869", "name": "restore-backup-minio", "reconcileID": "0e3651c6-32b6-4d87-aeca-db7a8fa8a967", "previous": "requested", "current": "error"} + kubectl_bin get psmdb-restore restore-backup-minio -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.QrL9vlAy3S ++ mktemp + local LAST_ERR=/tmp/tmp.cJazpmIqTb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-minio -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QrL9vlAy3S apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-minio","namespace":"demand-backup-physical-minio-native-tls-9869"},"spec":{"backupName":"backup-minio","clusterName":"some-name"}} creationTimestamp: "2026-05-21T08:18:59Z" generation: 1 name: restore-backup-minio namespace: demand-backup-physical-minio-native-tls-9869 resourceVersion: "1779351892751791006" uid: 1437575b-0676-4439-bb00-e548a8f6eaaa spec: backupName: backup-minio clusterName: some-name status: error: 'move to running state: wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644' pbmName: "2026-05-21T08:20:38.394170544Z" state: error + cat /tmp/tmp.cJazpmIqTb + rm /tmp/tmp.QrL9vlAy3S /tmp/tmp.cJazpmIqTb + return 0 + kubectl_bin describe psmdb-restore restore-backup-minio ++ mktemp + local LAST_OUT=/tmp/tmp.j1WJmhsr7Y ++ mktemp + local LAST_ERR=/tmp/tmp.HdhuWwiJbQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-minio + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j1WJmhsr7Y Name: restore-backup-minio Namespace: demand-backup-physical-minio-native-tls-9869 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2026-05-21T08:18:59Z Generation: 1 Resource Version: 1779351892751791006 UID: 1437575b-0676-4439-bb00-e548a8f6eaaa Spec: Backup Name: backup-minio Cluster Name: some-name Status: Error: move to running state: wait for nodes in rs: check heartbeat in .pbm.restore/2026-05-21T08:20:38.394170544Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-minio-native-tls-9869.svc.cluster.local:27017.hb: stuck, last beat ts: 1779351644 Pbm Name: 2026-05-21T08:20:38.394170544Z State: error Events: + cat /tmp/tmp.HdhuWwiJbQ + rm /tmp/tmp.j1WJmhsr7Y /tmp/tmp.HdhuWwiJbQ + return 0 + echo 'Restore object restore-backup-minio is in error state after 5 minutes.' Restore object restore-backup-minio is in error state after 5 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1