Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/logs/multi-storage.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + cluster=some-name + create_infra multi-storage-25897 + local ns=multi-storage-25897 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.DuXVA61Q2d ++ mktemp + local LAST_ERR=/tmp/tmp.ZXF09CUnco + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DuXVA61Q2d customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.ZXF09CUnco + rm /tmp/tmp.DuXVA61Q2d /tmp/tmp.ZXF09CUnco + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.xuGbZAx0uY ++ mktemp + local LAST_ERR=/tmp/tmp.p6Adx7Fy3d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xuGbZAx0uY customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.p6Adx7Fy3d + rm /tmp/tmp.xuGbZAx0uY /tmp/tmp.p6Adx7Fy3d + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.gOIZ59igit ++ mktemp + local LAST_ERR=/tmp/tmp.r0Ao7ptNAH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gOIZ59igit + cat /tmp/tmp.r0Ao7ptNAH + rm /tmp/tmp.gOIZ59igit /tmp/tmp.r0Ao7ptNAH + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.YbqCjyRq3U ++ mktemp + local LAST_ERR=/tmp/tmp.TYfRo3QqNU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YbqCjyRq3U + cat /tmp/tmp.TYfRo3QqNU + rm /tmp/tmp.YbqCjyRq3U /tmp/tmp.TYfRo3QqNU + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.XvArbsvyGG ++ mktemp + local LAST_ERR=/tmp/tmp.02jbqxQGHg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XvArbsvyGG clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.02jbqxQGHg + rm /tmp/tmp.XvArbsvyGG /tmp/tmp.02jbqxQGHg + return 0 + check_crd_for_deletion PR-2247-091270b4 + local git_tag=PR-2247-091270b4 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2247-091270b4/deploy/crd.yaml ++ /usr/sbin/sed s/---//g ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fZswGd5VPM +++ mktemp ++ local LAST_ERR=/tmp/tmp.GrJfl73X3R ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fZswGd5VPM ++ cat /tmp/tmp.GrJfl73X3R Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fZswGd5VPM ++ cat /tmp/tmp.GrJfl73X3R Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fZswGd5VPM ++ cat /tmp/tmp.GrJfl73X3R Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.fZswGd5VPM ++ cat /tmp/tmp.GrJfl73X3R Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.fZswGd5VPM /tmp/tmp.GrJfl73X3R ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' ++ mktemp + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.fMjksd1sRE + local LAST_OUT=/tmp/tmp.qFIGhZQtGJ ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.01hBmcsEP6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Xsh4jjTgBq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fMjksd1sRE + cat /tmp/tmp.01hBmcsEP6 + rm /tmp/tmp.fMjksd1sRE /tmp/tmp.01hBmcsEP6 + return 0 namespace "multi-storage-5684" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qFIGhZQtGJ namespace "psmdb-operator" deleted + cat /tmp/tmp.Xsh4jjTgBq + rm /tmp/tmp.qFIGhZQtGJ /tmp/tmp.Xsh4jjTgBq + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.SZB57r7zbO ++ mktemp + local LAST_ERR=/tmp/tmp.5HoWM2NzoO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SZB57r7zbO + cat /tmp/tmp.5HoWM2NzoO + rm /tmp/tmp.SZB57r7zbO /tmp/tmp.5HoWM2NzoO + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.kIrRCjyUge ++ mktemp + local LAST_ERR=/tmp/tmp.qDZpYHtlmc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kIrRCjyUge namespace/psmdb-operator created + cat /tmp/tmp.qDZpYHtlmc + rm /tmp/tmp.kIrRCjyUge /tmp/tmp.qDZpYHtlmc + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.0Vz5CwxvVz +++ mktemp ++ local LAST_ERR=/tmp/tmp.5oWoOVkwJD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0Vz5CwxvVz ++ cat /tmp/tmp.5oWoOVkwJD ++ rm /tmp/tmp.0Vz5CwxvVz /tmp/tmp.5oWoOVkwJD ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.C15PKrpDip ++ mktemp + local LAST_ERR=/tmp/tmp.p95GUR1ng4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.C15PKrpDip Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5" modified. + cat /tmp/tmp.p95GUR1ng4 + rm /tmp/tmp.C15PKrpDip /tmp/tmp.p95GUR1ng4 + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2247-091270b4' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2247-091270b4 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2FyMuJkCa6 ++ mktemp + local LAST_ERR=/tmp/tmp.Wl6SHXa4cg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2FyMuJkCa6 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.Wl6SHXa4cg + rm /tmp/tmp.2FyMuJkCa6 /tmp/tmp.Wl6SHXa4cg + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.JQdaXEFGLg ++ mktemp + local LAST_ERR=/tmp/tmp.oZOz3FXLVJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JQdaXEFGLg clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.oZOz3FXLVJ + rm /tmp/tmp.JQdaXEFGLg /tmp/tmp.oZOz3FXLVJ + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2247-091270b4") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kstUSN3TLU ++ mktemp + local LAST_ERR=/tmp/tmp.REYN8ze5DM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kstUSN3TLU deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.REYN8ze5DM + rm /tmp/tmp.kstUSN3TLU /tmp/tmp.REYN8ze5DM + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.lZJocGgYaQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.TzM1TvDMOn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lZJocGgYaQ ++ cat /tmp/tmp.TzM1TvDMOn ++ rm /tmp/tmp.lZJocGgYaQ /tmp/tmp.TzM1TvDMOn ++ return 0 + wait_operator_pod percona-server-mongodb-operator-64bc9b5fc9-7p5c7 + local pod=percona-server-mongodb-operator-64bc9b5fc9-7p5c7 + set +o xtrace waiting for pod/percona-server-mongodb-operator-64bc9b5fc9-7p5c7 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.saFYJbylrc +++ mktemp ++ local LAST_ERR=/tmp/tmp.lf0gNDmFE4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.saFYJbylrc ++ cat /tmp/tmp.lf0gNDmFE4 ++ rm /tmp/tmp.saFYJbylrc /tmp/tmp.lf0gNDmFE4 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-7p5c7 ++ mktemp + local LAST_OUT=/tmp/tmp.SW8ZRUiqdH ++ mktemp + local LAST_ERR=/tmp/tmp.EVDD8tH5Go + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-7p5c7 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SW8ZRUiqdH + cat /tmp/tmp.EVDD8tH5Go + rm /tmp/tmp.SW8ZRUiqdH /tmp/tmp.EVDD8tH5Go + return 0 2026-02-19T14:20:03.671Z INFO setup Manager starting up {"gitCommit": "091270b48773244359d74f74a10340fc1c0cff51", "gitBranch": "PR-2247-091270b4", "buildTime": "", "goVersion": "go1.25.7", "os": "linux", "arch": "amd64"} + create_namespace multi-storage-25897 + local namespace=multi-storage-25897 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces multi-storage-25897' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces multi-storage-25897+ xargs kubectl delete ns ----------------------------------------------------------------------------------- + kubectl_bin delete namespace multi-storage-25897 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.Bs1Dzk0HcZ ++ mktemp + local LAST_OUT=/tmp/tmp.fDB2sgDfWq + local LAST_ERR=/tmp/tmp.IgizAhQLXJ + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ mktemp + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.GgxB4kXxHX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace multi-storage-25897 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bs1Dzk0HcZ + cat /tmp/tmp.IgizAhQLXJ + rm /tmp/tmp.Bs1Dzk0HcZ /tmp/tmp.IgizAhQLXJ + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fDB2sgDfWq + cat /tmp/tmp.GgxB4kXxHX + rm /tmp/tmp.fDB2sgDfWq /tmp/tmp.GgxB4kXxHX + return 0 + kubectl_bin wait --for=delete namespace multi-storage-25897 ++ mktemp + local LAST_OUT=/tmp/tmp.Oc5wj5iIyz ++ mktemp + local LAST_ERR=/tmp/tmp.MCJQ2zLf5A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace multi-storage-25897 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Oc5wj5iIyz + cat /tmp/tmp.MCJQ2zLf5A + rm /tmp/tmp.Oc5wj5iIyz /tmp/tmp.MCJQ2zLf5A + return 0 + desc 'create namespace multi-storage-25897' + set +o xtrace ----------------------------------------------------------------------------------- create namespace multi-storage-25897 ----------------------------------------------------------------------------------- + kubectl_bin create namespace multi-storage-25897 ++ mktemp + local LAST_OUT=/tmp/tmp.K246tz0mMQ ++ mktemp + local LAST_ERR=/tmp/tmp.TNcijapbJ4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace multi-storage-25897 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.K246tz0mMQ namespace/multi-storage-25897 created + cat /tmp/tmp.TNcijapbJ4 + rm /tmp/tmp.K246tz0mMQ /tmp/tmp.TNcijapbJ4 + return 0 + set_kube_ctx multi-storage-25897 + local namespace=multi-storage-25897 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.09m3mYyUTU +++ mktemp ++ local LAST_ERR=/tmp/tmp.x1A0rn6EkM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.09m3mYyUTU ++ cat /tmp/tmp.x1A0rn6EkM ++ rm /tmp/tmp.09m3mYyUTU /tmp/tmp.x1A0rn6EkM ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5 --namespace=multi-storage-25897 ++ mktemp + local LAST_OUT=/tmp/tmp.R5SiziQsY9 ++ mktemp + local LAST_ERR=/tmp/tmp.Z4iacLuJLM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5 --namespace=multi-storage-25897 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R5SiziQsY9 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster5" modified. + cat /tmp/tmp.Z4iacLuJLM + rm /tmp/tmp.R5SiziQsY9 /tmp/tmp.Z4iacLuJLM + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Thu Feb 19 14:20:41 2026 NAMESPACE: multi-storage-25897 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.multi-storage-25897.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace multi-storage-25897 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace multi-storage-25897 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace multi-storage-25897 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace multi-storage-25897 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QYVuGKxZrJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.o2PI9BxbZK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QYVuGKxZrJ ++ cat /tmp/tmp.o2PI9BxbZK ++ rm /tmp/tmp.QYVuGKxZrJ /tmp/tmp.o2PI9BxbZK ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-vmrxh + wait_pod minio-service-6d5f646cdc-vmrxh + local pod=minio-service-6d5f646cdc-vmrxh + set +o xtrace waiting for pod/minio-service-6d5f646cdc-vmrxh to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.multi-storage-25897.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.wqrg5ABoSV ++ mktemp + local LAST_ERR=/tmp/tmp.o23ug6avAg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wqrg5ABoSV make_bucket: operator-testing pod "aws-cli" deleted from multi-storage-25897 namespace + cat /tmp/tmp.o23ug6avAg All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-25897 + rm /tmp/tmp.wqrg5ABoSV /tmp/tmp.o23ug6avAg + return 0 + endpoint=http://minio-service:9000 + create_minio_bucket operator-testing-2 http://minio-service:9000 + local bucket=operator-testing-2 + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2' ++ mktemp + local LAST_OUT=/tmp/tmp.WFwApdQLkH ++ mktemp + local LAST_ERR=/tmp/tmp.w9cdCuF4ry + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-2' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WFwApdQLkH make_bucket: operator-testing-2 pod "aws-cli" deleted from multi-storage-25897 namespace + cat /tmp/tmp.w9cdCuF4ry + rm /tmp/tmp.WFwApdQLkH /tmp/tmp.w9cdCuF4ry + return 0 + create_minio_bucket operator-testing-3 http://minio-service:9000 + local bucket=operator-testing-3 + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3' ++ mktemp + local LAST_OUT=/tmp/tmp.cYrlZDdlmh ++ mktemp + local LAST_ERR=/tmp/tmp.JtImk3inGx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing-3' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cYrlZDdlmh make_bucket: operator-testing-3 pod "aws-cli" deleted from multi-storage-25897 namespace + cat /tmp/tmp.JtImk3inGx All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_multi-storage-25897 + rm /tmp/tmp.cYrlZDdlmh /tmp/tmp.JtImk3inGx + return 0 + log 'create secrets and start client' + set +o xtrace [2026-02-19T14:21:27+0000] create secrets and start client + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ERRRxQgMja ++ mktemp + local LAST_ERR=/tmp/tmp.ow1ZCURlrX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ERRRxQgMja secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.ow1ZCURlrX + rm /tmp/tmp.ERRRxQgMja /tmp/tmp.ow1ZCURlrX + return 0 + log 'create PSMDB cluster: some-name' + set +o xtrace [2026-02-19T14:21:29+0000] create PSMDB cluster: some-name + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2247-091270b4"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/multi-storage-25897/g + local LAST_OUT=/tmp/tmp.DzkssPiPsN ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.7KPPoRqR48 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DzkssPiPsN perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.7KPPoRqR48 + rm /tmp/tmp.DzkssPiPsN /tmp/tmp.7KPPoRqR48 + return 0 + log 'check if all 3 Pods started' + set +o xtrace [2026-02-19T14:21:31+0000] check if all 3 Pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4MMw514VuA +++ mktemp ++ local LAST_ERR=/tmp/tmp.YEU17VgsDw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4MMw514VuA ++ cat /tmp/tmp.YEU17VgsDw ++ rm /tmp/tmp.4MMw514VuA /tmp/tmp.YEU17VgsDw ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RhPA5XvM69 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qFydydeHx5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RhPA5XvM69 ++ cat /tmp/tmp.qFydydeHx5 ++ rm /tmp/tmp.RhPA5XvM69 /tmp/tmp.qFydydeHx5 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DECp8lAI0q +++ mktemp ++ local LAST_ERR=/tmp/tmp.QlxnBuki06 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DECp8lAI0q ++ cat /tmp/tmp.QlxnBuki06 ++ rm /tmp/tmp.DECp8lAI0q /tmp/tmp.QlxnBuki06 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.... + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-02-19T14:22:50.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-02-19T14:22:59.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-02-19T14:23:01.000+0000 I listening for the commands + create_user + log 'create user myApp' + set +o xtrace [2026-02-19T14:23:06+0000] create user myApp + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.multi-storage-25897 mongodb + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.85Cf3c5ve5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VcAnpIipAJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.85Cf3c5ve5 ++ cat /tmp/tmp.VcAnpIipAJ ++ rm /tmp/tmp.85Cf3c5ve5 /tmp/tmp.VcAnpIipAJ ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.qA4MQxveWA ++ mktemp + local LAST_ERR=/tmp/tmp.rCvcdSszv3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qA4MQxveWA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("377ca076-8ffb-456c-9849-d458d15d40a3") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.rCvcdSszv3 + rm /tmp/tmp.qA4MQxveWA /tmp/tmp.rCvcdSszv3 + return 0 + write_document 100500 + local x=100500 + local cmp_postfix= + log 'write document: 100500' + set +o xtrace [2026-02-19T14:23:08+0000] write document: 100500 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PnZ6ldlMwh +++ mktemp ++ local LAST_ERR=/tmp/tmp.S4g6FIbJPZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PnZ6ldlMwh ++ cat /tmp/tmp.S4g6FIbJPZ ++ rm /tmp/tmp.PnZ6ldlMwh /tmp/tmp.S4g6FIbJPZ ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.3QNbo12Z0j ++ mktemp + local LAST_ERR=/tmp/tmp.ypazYY537t + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3QNbo12Z0j Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("e18bd3ad-1288-4447-a5e2-66b5210038fb") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ypazYY537t + rm /tmp/tmp.3QNbo12Z0j /tmp/tmp.ypazYY537t + return 0 + run_backup minio-1 backup-minio-1 logical + local storage=minio-1 + local name=backup-minio-1 + local type=logical + log 'creating backup: backup-minio-1 (logical)' + set +o xtrace [2026-02-19T14:23:11+0000] creating backup: backup-minio-1 (logical) + yq eval '.metadata.name = "backup-minio-1" | .spec.storageName = "minio-1" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AbAtnWwBTJ ++ mktemp + local LAST_ERR=/tmp/tmp.xOCeSelmsH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AbAtnWwBTJ perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.xOCeSelmsH + rm /tmp/tmp.AbAtnWwBTJ /tmp/tmp.xOCeSelmsH + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state.......OK + run_backup minio-2 backup-minio-2 logical + local storage=minio-2 + local name=backup-minio-2 + local type=logical + log 'creating backup: backup-minio-2 (logical)' + set +o xtrace [2026-02-19T14:23:25+0000] creating backup: backup-minio-2 (logical) + yq eval '.metadata.name = "backup-minio-2" | .spec.storageName = "minio-2" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.lNvEgdBW29 ++ mktemp + local LAST_ERR=/tmp/tmp.U7ZczWgNJt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lNvEgdBW29 perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.U7ZczWgNJt + rm /tmp/tmp.lNvEgdBW29 /tmp/tmp.U7ZczWgNJt + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state.......OK + run_backup minio-3 backup-minio-3 physical + local storage=minio-3 + local name=backup-minio-3 + local type=physical + log 'creating backup: backup-minio-3 (physical)' + set +o xtrace [2026-02-19T14:23:39+0000] creating backup: backup-minio-3 (physical) + yq eval '.metadata.name = "backup-minio-3" | .spec.storageName = "minio-3" | .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.jtFRWQFAVA ++ mktemp + local LAST_ERR=/tmp/tmp.g88gocoqbs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jtFRWQFAVA perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created + cat /tmp/tmp.g88gocoqbs + rm /tmp/tmp.jtFRWQFAVA /tmp/tmp.g88gocoqbs + return 0 + wait_backup backup-minio-3 + local backup_name=backup-minio-3 + local target_state=ready + set +o xtrace waiting for backup-minio-3 to reach ready state.......OK + write_document 100501 + local x=100501 + local cmp_postfix= + log 'write document: 100501' + set +o xtrace [2026-02-19T14:23:52+0000] write document: 100501 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WF1Qa3SiLQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.foFv8haCPD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WF1Qa3SiLQ ++ cat /tmp/tmp.foFv8haCPD ++ rm /tmp/tmp.WF1Qa3SiLQ /tmp/tmp.foFv8haCPD ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.XDH63LQGQz ++ mktemp + local LAST_ERR=/tmp/tmp.XLCQ4z1CFD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XDH63LQGQz Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("a91cc0a6-8cef-4392-890e-cf0976f68c8e") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.XLCQ4z1CFD + rm /tmp/tmp.XDH63LQGQz /tmp/tmp.XLCQ4z1CFD + return 0 + write_document 100502 + local x=100502 + local cmp_postfix= + log 'write document: 100502' + set +o xtrace [2026-02-19T14:23:55+0000] write document: 100502 + run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100502 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kw2N1GkRao +++ mktemp ++ local LAST_ERR=/tmp/tmp.jyXoXtIi8m ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kw2N1GkRao ++ cat /tmp/tmp.jyXoXtIi8m ++ rm /tmp/tmp.kw2N1GkRao /tmp/tmp.jyXoXtIi8m ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.UQB6gRb22A ++ mktemp + local LAST_ERR=/tmp/tmp.GpE1HILl6t + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UQB6gRb22A Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("5d39f00a-3b4d-4e9e-b445-472010378ac9") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.GpE1HILl6t + rm /tmp/tmp.UQB6gRb22A /tmp/tmp.GpE1HILl6t + return 0 + write_document 100503 + local x=100503 + local cmp_postfix= + log 'write document: 100503' + set +o xtrace [2026-02-19T14:23:57+0000] write document: 100503 + run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100503 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LXm37stRVH +++ mktemp ++ local LAST_ERR=/tmp/tmp.7o0zYolJde ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LXm37stRVH ++ cat /tmp/tmp.7o0zYolJde ++ rm /tmp/tmp.LXm37stRVH /tmp/tmp.7o0zYolJde ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.K9IakOeeqN ++ mktemp + local LAST_ERR=/tmp/tmp.peZwvN1YxC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.K9IakOeeqN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("080e182a-769c-400c-840d-f9f0215cfefe") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.peZwvN1YxC + rm /tmp/tmp.K9IakOeeqN /tmp/tmp.peZwvN1YxC + return 0 + write_document 100504 + local x=100504 + local cmp_postfix= + log 'write document: 100504' + set +o xtrace [2026-02-19T14:23:59+0000] write document: 100504 + run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100504 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Zcf06SazC +++ mktemp ++ local LAST_ERR=/tmp/tmp.DaWp1h49Xx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6Zcf06SazC ++ cat /tmp/tmp.DaWp1h49Xx ++ rm /tmp/tmp.6Zcf06SazC /tmp/tmp.DaWp1h49Xx ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.vSEBGQm645 ++ mktemp + local LAST_ERR=/tmp/tmp.BpsSHaaazP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vSEBGQm645 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("83073d4c-fb7f-43f2-aaaa-e41b13f58d51") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.BpsSHaaazP + rm /tmp/tmp.vSEBGQm645 /tmp/tmp.BpsSHaaazP + return 0 + write_document 100505 + local x=100505 + local cmp_postfix= + log 'write document: 100505' + set +o xtrace [2026-02-19T14:24:01+0000] write document: 100505 + run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.insert({ x: 100505 })' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Bt61WkuueA +++ mktemp ++ local LAST_ERR=/tmp/tmp.8iZ2pzmSp4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Bt61WkuueA ++ cat /tmp/tmp.8iZ2pzmSp4 ++ rm /tmp/tmp.Bt61WkuueA /tmp/tmp.8iZ2pzmSp4 ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.k7pDxA9aSW ++ mktemp + local LAST_ERR=/tmp/tmp.L6z1EWZkXR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.k7pDxA9aSW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f989bd2f-2e44-4d98-8fc0-7b16b534834e") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.L6z1EWZkXR + rm /tmp/tmp.k7pDxA9aSW /tmp/tmp.L6z1EWZkXR + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.multi-storage-25897 + local command=find + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-02-19T14:24:03+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0OXw3uu25x +++ mktemp ++ local LAST_ERR=/tmp/tmp.mmEyUB2U1S ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0OXw3uu25x ++ cat /tmp/tmp.mmEyUB2U1S ++ rm /tmp/tmp.0OXw3uu25x /tmp/tmp.mmEyUB2U1S ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.cTyMbL9FQx ++ mktemp + local LAST_ERR=/tmp/tmp.iRMr3shLwn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cTyMbL9FQx + cat /tmp/tmp.iRMr3shLwn + rm /tmp/tmp.cTyMbL9FQx /tmp/tmp.iRMr3shLwn + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/compare/find.json /tmp/tmp.OtgHopOuH7/find + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-02-19T14:24:06+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 + wait_for_restorable_time backup-minio-3 + local backup_name=backup-minio-3 ++ get_latest_restorable_time backup-minio-3 ++ local backup_name=backup-minio-3 +++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml +++ yq .status.latestRestorableTime ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qct1Dq8ag1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.haKuqn33Va +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-3 -o yaml +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qct1Dq8ag1 +++ cat /tmp/tmp.haKuqn33Va +++ rm /tmp/tmp.qct1Dq8ag1 /tmp/tmp.haKuqn33Va +++ return 0 ++ local time=2026-02-19T14:24:58Z ++ [[ 2026-02-19T14:24:58Z == \n\u\l\l ]] ++ echo -n 2026-02-19T14:24:58Z + local latest_restorable_time=2026-02-19T14:24:58Z + local retries=0 + echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time' waiting for psmdb-backup/backup-minio-3's latest restorable time+ [[ 2026-02-19T14:24:58Z != \n\u\l\l ]] + echo '.OK latestRestorableTime: 2026-02-19T14:24:58Z' .OK latestRestorableTime: 2026-02-19T14:24:58Z + wait_for_advanced_restorable_time backup-minio-3 + local backup_name=backup-minio-3 +++ get_latest_restorable_time backup-minio-3 +++ local backup_name=backup-minio-3 ++++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml ++++ yq .status.latestRestorableTime +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.JIgdoLKYIX +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ZX1j6gWdvp ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl get psmdb-backup backup-minio-3 -o yaml ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.JIgdoLKYIX ++++ cat /tmp/tmp.ZX1j6gWdvp ++++ rm /tmp/tmp.JIgdoLKYIX /tmp/tmp.ZX1j6gWdvp ++++ return 0 +++ local time=2026-02-19T14:24:58Z +++ [[ 2026-02-19T14:24:58Z == \n\u\l\l ]] +++ echo -n 2026-02-19T14:24:58Z ++ datetime_to_timestamp 2026-02-19T14:24:58Z ++ local datetime=2026-02-19T14:24:58Z ++ TZ=UTC ++ /usr/sbin/date -d2026-02-19T14:24:58Z +%s + local latest_restorable_time=1771511098 ++ get_backup_last_write backup-minio-3 ++ local backup_name=backup-minio-3 +++ yq .status.pbmName +++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml ++++ mktemp +++ local LAST_OUT=/tmp/tmp.glyzjOQ3DH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fcXxSJCit2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-3 -o yaml +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.glyzjOQ3DH +++ cat /tmp/tmp.fcXxSJCit2 +++ rm /tmp/tmp.glyzjOQ3DH /tmp/tmp.fcXxSJCit2 +++ return 0 ++ local pbm_name=2026-02-19T14:23:40Z ++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm list -o json ++ jq --arg v 2026-02-19T14:23:40Z '.snapshots[] | select(.name==$v) | .restoreTo' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oHz5EqC4KC +++ mktemp ++ local LAST_ERR=/tmp/tmp.7ReCWx7SFh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm list -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oHz5EqC4KC ++ cat /tmp/tmp.7ReCWx7SFh ++ rm /tmp/tmp.oHz5EqC4KC /tmp/tmp.7ReCWx7SFh ++ return 0 + local latest_write=1771511022 + local retries=0 ++ timestamp_to_datetime 1771511022 ++ local ts=1771511022 ++ TZ=UTC ++ /usr/sbin/date -d@1771511022 +%Y-%m-%dT%H:%M:%S%Z + echo -n 'waiting for psmdb-backup/backup-minio-3'\''s latest restorable time to advance its latest write (2026-02-19T14:23:42UTC)' waiting for psmdb-backup/backup-minio-3's latest restorable time to advance its latest write (2026-02-19T14:23:42UTC)+ [[ 1771511098 -gt 1771511022 ]] ++ timestamp_to_datetime 1771511098 ++ local ts=1771511098 ++ TZ=UTC ++ /usr/sbin/date -d@1771511098 +%Y-%m-%dT%H:%M:%S%Z + echo '.OK latestRestorableTime: 2026-02-19T14:24:58UTC' .OK latestRestorableTime: 2026-02-19T14:24:58UTC ++ get_latest_restorable_time backup-minio-3 ++ local backup_name=backup-minio-3 +++ kubectl_bin get psmdb-backup backup-minio-3 -o yaml +++ yq .status.latestRestorableTime ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kKocXAg9mI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bCcG4LFnq2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-3 -o yaml +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kKocXAg9mI +++ cat /tmp/tmp.bCcG4LFnq2 +++ rm /tmp/tmp.kKocXAg9mI /tmp/tmp.bCcG4LFnq2 +++ return 0 ++ local time=2026-02-19T14:24:58Z ++ [[ 2026-02-19T14:24:58Z == \n\u\l\l ]] ++ echo -n 2026-02-19T14:24:58Z + restore_time=2026-02-19T14:24:58Z ++ format_pitr_target 2026-02-19T14:24:58Z ++ local target=2026-02-19T14:24:58Z ++ echo 2026-02-19T14:24:58Z ++ sed 's/T/ /' ++ sed s/Z// + check_recovery backup-minio-3 '2026-02-19 14:24:58' + local backup_name=backup-minio-3 + local 'restore_time=2026-02-19 14:24:58' + local cmp_postfix= + drop_collection myApp test myApp myPass + local db=myApp + local collection=test + local user=myApp + local pass=myPass + log 'dropping collection: myApp.test' + set +o xtrace [2026-02-19T14:25:21+0000] dropping collection: myApp.test + run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.multi-storage-25897 mongodb + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.multi-storage-25897 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.multi-storage-25897 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ko8C1ewuaD +++ mktemp ++ local LAST_ERR=/tmp/tmp.CLhlR4gITF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ko8C1ewuaD ++ cat /tmp/tmp.CLhlR4gITF ++ rm /tmp/tmp.ko8C1ewuaD /tmp/tmp.CLhlR4gITF ++ return 0 + local client_container=psmdb-client-86cb5d8484-v7878 + kubectl_bin exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kJvub4Fiqz ++ mktemp + local LAST_ERR=/tmp/tmp.pnSQffB1x9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-86cb5d8484-v7878 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.multi-storage-25897.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kJvub4Fiqz Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0.multi-storage-25897.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7a8badcc-8075-4d94-ab01-be1467a29ae1") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.pnSQffB1x9 + rm /tmp/tmp.kJvub4Fiqz /tmp/tmp.pnSQffB1x9 + return 0 + log 'creating restore: restore-backup-minio-3 restore_to: 2026-02-19 14:24:58' + set +o xtrace [2026-02-19T14:25:23+0000] creating restore: restore-backup-minio-3 restore_to: 2026-02-19 14:24:58 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/multi-storage/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-3/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio-3/' + /usr/sbin/sed -e 's/date:/date: 2026-02-19 14:24:58/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.87iGjV8Fd8 ++ mktemp + local LAST_ERR=/tmp/tmp.rqSCLfeQe3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.87iGjV8Fd8 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created + cat /tmp/tmp.rqSCLfeQe3 + rm /tmp/tmp.87iGjV8Fd8 /tmp/tmp.rqSCLfeQe3 + return 0 + wait_restore backup-minio-3 some-name requested 0 900 1 + local backup_name=backup-minio-3 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=900 + local ok_if_ready=1 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-3 object to be created.OK Waiting psmdb-restore/restore-backup-minio-3 to reach state "requested" ............... ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.cGaloNipLQ ++ mktemp + local LAST_ERR=/tmp/tmp.Tj8pE7KnWe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.cGaloNipLQ + cat /tmp/tmp.Tj8pE7KnWe Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.cGaloNipLQ + cat /tmp/tmp.Tj8pE7KnWe Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.cGaloNipLQ + cat /tmp/tmp.Tj8pE7KnWe Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 8 + cat /tmp/tmp.cGaloNipLQ + cat /tmp/tmp.Tj8pE7KnWe Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + rm /tmp/tmp.cGaloNipLQ /tmp/tmp.Tj8pE7KnWe + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-0' failed to get /tmp/pbm-agent.log from some-name-rs0-0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.fKYkW6ZpeY ++ mktemp + local LAST_ERR=/tmp/tmp.wBSMsvBDJL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.fKYkW6ZpeY + cat /tmp/tmp.wBSMsvBDJL Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.fKYkW6ZpeY + cat /tmp/tmp.wBSMsvBDJL Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.fKYkW6ZpeY + cat /tmp/tmp.wBSMsvBDJL Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 8 + cat /tmp/tmp.fKYkW6ZpeY + cat /tmp/tmp.wBSMsvBDJL Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + rm /tmp/tmp.fKYkW6ZpeY /tmp/tmp.wBSMsvBDJL + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-1' failed to get /tmp/pbm-agent.log from some-name-rs0-1 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.Wqx9cC5uko ++ mktemp + local LAST_ERR=/tmp/tmp.kJewH8C80l + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Wqx9cC5uko + cat /tmp/tmp.kJewH8C80l Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Wqx9cC5uko + cat /tmp/tmp.kJewH8C80l Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Wqx9cC5uko + cat /tmp/tmp.kJewH8C80l Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 8 + cat /tmp/tmp.Wqx9cC5uko + cat /tmp/tmp.kJewH8C80l Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + rm /tmp/tmp.Wqx9cC5uko /tmp/tmp.kJewH8C80l + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-2' failed to get /tmp/pbm-agent.log from some-name-rs0-2 + grep restore-backup-minio-3 + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.W4U9AyJZJB +++ mktemp ++ local LAST_ERR=/tmp/tmp.26PLTIda8N ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W4U9AyJZJB ++ cat /tmp/tmp.26PLTIda8N ++ rm /tmp/tmp.W4U9AyJZJB /tmp/tmp.26PLTIda8N ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-7p5c7 ++ mktemp + local LAST_OUT=/tmp/tmp.EnFoYTgcTc ++ mktemp + local LAST_ERR=/tmp/tmp.tKvP4llcen + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-7p5c7 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EnFoYTgcTc + cat /tmp/tmp.tKvP4llcen + rm /tmp/tmp.EnFoYTgcTc /tmp/tmp.tKvP4llcen + return 0 2026-02-19T14:37:26.346Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "79363a3d-71af-4957-b611-ea4c04e7da69", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:26.346Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "79363a3d-71af-4957-b611-ea4c04e7da69", "ready": false} 2026-02-19T14:37:31.347Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "05a70c98-5c9f-4589-a921-3839fb4d4124", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:31.348Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "05a70c98-5c9f-4589-a921-3839fb4d4124", "ready": false} 2026-02-19T14:37:36.348Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c73ce519-8f5d-4dbf-8f82-0a985807ac99", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:36.349Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c73ce519-8f5d-4dbf-8f82-0a985807ac99", "ready": false} 2026-02-19T14:37:41.350Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "9bccb31c-cee6-4b15-9de9-8cb196524a6a", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:41.350Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "9bccb31c-cee6-4b15-9de9-8cb196524a6a", "ready": false} 2026-02-19T14:37:46.351Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "4147a87e-decd-45f3-af81-8a0a89547c07", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:46.351Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "4147a87e-decd-45f3-af81-8a0a89547c07", "ready": false} 2026-02-19T14:37:51.353Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b3e9fbb6-ff67-4e02-9c01-61ff143f1e26", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:51.353Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b3e9fbb6-ff67-4e02-9c01-61ff143f1e26", "ready": false} 2026-02-19T14:37:56.355Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "4e746491-8050-4532-ac32-fb6f8fc5a1b4", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:37:56.355Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "4e746491-8050-4532-ac32-fb6f8fc5a1b4", "ready": false} 2026-02-19T14:38:01.356Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "99e18dd0-55a3-47a5-9a75-984f1ec86969", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:01.357Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "99e18dd0-55a3-47a5-9a75-984f1ec86969", "ready": false} 2026-02-19T14:38:06.357Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c02ab4e7-99b8-4a3d-910e-f32e0d991bfd", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:06.358Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c02ab4e7-99b8-4a3d-910e-f32e0d991bfd", "ready": false} 2026-02-19T14:38:11.359Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7a75b92a-3c57-412d-837d-018e6435a6e6", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:11.360Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7a75b92a-3c57-412d-837d-018e6435a6e6", "ready": false} 2026-02-19T14:38:16.361Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "094995d1-9c87-4c50-afa1-70aaceb4d452", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:16.361Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "094995d1-9c87-4c50-afa1-70aaceb4d452", "ready": false} 2026-02-19T14:38:21.363Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d254d1c3-8876-4dc4-a788-32dfc452d087", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:21.364Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d254d1c3-8876-4dc4-a788-32dfc452d087", "ready": false} 2026-02-19T14:38:26.364Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b6ab27f6-024a-4482-bdea-d0ce84e7d698", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:26.365Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b6ab27f6-024a-4482-bdea-d0ce84e7d698", "ready": false} 2026-02-19T14:38:31.365Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "68b6ed9b-9711-4688-be03-1584227889e2", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:31.366Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "68b6ed9b-9711-4688-be03-1584227889e2", "ready": false} 2026-02-19T14:38:36.366Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "eb72fafd-6aca-410b-9d3d-20032d60b988", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:36.367Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "eb72fafd-6aca-410b-9d3d-20032d60b988", "ready": false} 2026-02-19T14:38:41.368Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "412502e3-f961-48ac-b878-1b483668fdad", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:41.369Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "412502e3-f961-48ac-b878-1b483668fdad", "ready": false} 2026-02-19T14:38:46.371Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d3437248-1bac-4206-affd-113ddb6f123b", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:46.371Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d3437248-1bac-4206-affd-113ddb6f123b", "ready": false} 2026-02-19T14:38:51.372Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6033d04d-55a0-484c-baeb-345055f88027", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:51.373Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6033d04d-55a0-484c-baeb-345055f88027", "ready": false} 2026-02-19T14:38:56.374Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "0804c143-ab67-4952-86e5-a84f923ef0b9", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:38:56.374Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "0804c143-ab67-4952-86e5-a84f923ef0b9", "ready": false} 2026-02-19T14:39:01.376Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7cd65d56-5f5b-4bad-b3e0-5bdb11e12fae", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:01.376Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7cd65d56-5f5b-4bad-b3e0-5bdb11e12fae", "ready": false} 2026-02-19T14:39:06.377Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f0200a0e-9f31-4331-8dc3-45f8b92a1a29", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:06.378Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f0200a0e-9f31-4331-8dc3-45f8b92a1a29", "ready": false} 2026-02-19T14:39:11.379Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "e621d01c-4c3c-4c0f-ba42-a7e7d4728a36", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:11.380Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "e621d01c-4c3c-4c0f-ba42-a7e7d4728a36", "ready": false} 2026-02-19T14:39:16.381Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "2f313038-61c3-4f08-ac9f-c0d3395d57a1", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:16.381Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "2f313038-61c3-4f08-ac9f-c0d3395d57a1", "ready": false} 2026-02-19T14:39:21.382Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "5e9ab4a7-d768-46ea-88fc-d56b0fa4a075", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:21.383Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "5e9ab4a7-d768-46ea-88fc-d56b0fa4a075", "ready": false} 2026-02-19T14:39:26.384Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6f5568b0-d2ed-4e9d-8ea7-33c6345ca1be", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:26.384Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6f5568b0-d2ed-4e9d-8ea7-33c6345ca1be", "ready": false} 2026-02-19T14:39:31.385Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "016be87a-5ec7-4471-aab7-1d09dcfc1730", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:31.386Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "016be87a-5ec7-4471-aab7-1d09dcfc1730", "ready": false} 2026-02-19T14:39:36.387Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "567b0921-6b6b-4791-8ea9-183b8cf98ff7", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:36.388Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "567b0921-6b6b-4791-8ea9-183b8cf98ff7", "ready": false} 2026-02-19T14:39:41.388Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "8408d6bd-1d76-413e-92d7-32f23c6cd4f2", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:41.389Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "8408d6bd-1d76-413e-92d7-32f23c6cd4f2", "ready": false} 2026-02-19T14:39:46.390Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6a950840-b4c7-454e-83ac-8c8f33cc75c4", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:46.391Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6a950840-b4c7-454e-83ac-8c8f33cc75c4", "ready": false} 2026-02-19T14:39:51.392Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "689818f3-20fb-4a9c-bdec-8c980e65659f", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:51.393Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "689818f3-20fb-4a9c-bdec-8c980e65659f", "ready": false} 2026-02-19T14:39:56.394Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6f47e3ae-5513-4bc2-a624-622671577a66", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:39:56.395Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "6f47e3ae-5513-4bc2-a624-622671577a66", "ready": false} 2026-02-19T14:40:01.396Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7f43e2fb-98d0-47db-9af9-090d9bb6c600", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:01.397Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7f43e2fb-98d0-47db-9af9-090d9bb6c600", "ready": false} 2026-02-19T14:40:06.397Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "730e6687-f2e2-4a05-90d8-6b7ef73289cc", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:06.398Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "730e6687-f2e2-4a05-90d8-6b7ef73289cc", "ready": false} 2026-02-19T14:40:11.399Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c21505ac-8b55-43fa-8a3c-160e75b0c33e", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:11.400Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c21505ac-8b55-43fa-8a3c-160e75b0c33e", "ready": false} 2026-02-19T14:40:16.401Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "cb497b32-8489-49d9-a81a-3c987d239662", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:16.402Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "cb497b32-8489-49d9-a81a-3c987d239662", "ready": false} 2026-02-19T14:40:21.403Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "62fef177-faad-46fb-8330-663a75c820e5", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:21.403Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "62fef177-faad-46fb-8330-663a75c820e5", "ready": false} 2026-02-19T14:40:26.404Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c8a1d0c9-ab8d-4ec7-b0cd-b7649777abf3", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:26.405Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "c8a1d0c9-ab8d-4ec7-b0cd-b7649777abf3", "ready": false} 2026-02-19T14:40:31.407Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "bafdf849-0b54-4cc0-bbfd-1bce06a93e28", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:31.407Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "bafdf849-0b54-4cc0-bbfd-1bce06a93e28", "ready": false} 2026-02-19T14:40:36.408Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d2287178-e44e-44f5-b576-a9ae1021392c", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:36.409Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "d2287178-e44e-44f5-b576-a9ae1021392c", "ready": false} 2026-02-19T14:40:41.410Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "007ea110-046d-4c36-aab2-54a80bb10ff0", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:41.410Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "007ea110-046d-4c36-aab2-54a80bb10ff0", "ready": false} 2026-02-19T14:40:46.411Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "83328165-cec8-42a7-8914-ba70ce6ce2e2", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:46.412Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "83328165-cec8-42a7-8914-ba70ce6ce2e2", "ready": false} 2026-02-19T14:40:51.413Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "e258eb2e-6f7b-441e-a9ca-34f87aaa315d", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:51.413Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "e258eb2e-6f7b-441e-a9ca-34f87aaa315d", "ready": false} 2026-02-19T14:40:56.414Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f14d2bdd-061b-4200-a8ea-22e2266de115", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:40:56.415Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f14d2bdd-061b-4200-a8ea-22e2266de115", "ready": false} 2026-02-19T14:41:01.416Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7f62079a-f3ec-46b2-9ebe-7402edb86d87", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:01.417Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "7f62079a-f3ec-46b2-9ebe-7402edb86d87", "ready": false} 2026-02-19T14:41:06.418Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f8a06ed4-fe3d-40b2-b75a-55dd6d6f7ee1", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:06.419Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "f8a06ed4-fe3d-40b2-b75a-55dd6d6f7ee1", "ready": false} 2026-02-19T14:41:11.420Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b1217b13-3c64-46c2-a8a0-eb735b361276", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:11.421Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "b1217b13-3c64-46c2-a8a0-eb735b361276", "ready": false} 2026-02-19T14:41:16.421Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "ab4c1e2a-906e-4254-aa80-fabc9ba73837", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:16.422Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "ab4c1e2a-906e-4254-aa80-fabc9ba73837", "ready": false} 2026-02-19T14:41:21.423Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "169a50f8-9493-4851-b7f4-9e699d3e2c3d", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:21.424Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "169a50f8-9493-4851-b7f4-9e699d3e2c3d", "ready": false} 2026-02-19T14:41:26.424Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "22a5ad09-17d3-48f8-9e55-27c7f6eae883", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:26.425Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "22a5ad09-17d3-48f8-9e55-27c7f6eae883", "ready": false} 2026-02-19T14:41:31.426Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "261c695b-6cc6-40ec-97ea-a1bc149a95d3", "cluster": "some-name", "namespace": "multi-storage-25897"} 2026-02-19T14:41:31.427Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-3","namespace":"multi-storage-25897"}, "namespace": "multi-storage-25897", "name": "restore-backup-minio-3", "reconcileID": "261c695b-6cc6-40ec-97ea-a1bc149a95d3", "ready": false} + kubectl_bin get psmdb-restore restore-backup-minio-3 -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.aatPeRFd1n ++ mktemp + local LAST_ERR=/tmp/tmp.SdRZpM0EQX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-minio-3 -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aatPeRFd1n apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-minio-3","namespace":"multi-storage-25897"},"spec":{"backupName":"backup-minio-3","clusterName":"some-name","pitr":{"date":"2026-02-19 14:24:58","type":"date"}}} creationTimestamp: "2026-02-19T14:25:25Z" generation: 1 name: restore-backup-minio-3 namespace: multi-storage-25897 resourceVersion: "1771511126019631020" uid: 3679983e-ae5c-422f-8b1c-1c5f975f2ecf spec: backupName: backup-minio-3 clusterName: some-name pitr: date: "2026-02-19 14:24:58" type: date status: pitrTarget: 2026-02-19T14:24:58 state: waiting + cat /tmp/tmp.SdRZpM0EQX + rm /tmp/tmp.aatPeRFd1n /tmp/tmp.SdRZpM0EQX + return 0 + kubectl_bin describe psmdb-restore restore-backup-minio-3 ++ mktemp + local LAST_OUT=/tmp/tmp.REHQx5s1KR ++ mktemp + local LAST_ERR=/tmp/tmp.XDquXbwIc6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-minio-3 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.REHQx5s1KR Name: restore-backup-minio-3 Namespace: multi-storage-25897 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2026-02-19T14:25:25Z Generation: 1 Resource Version: 1771511126019631020 UID: 3679983e-ae5c-422f-8b1c-1c5f975f2ecf Spec: Backup Name: backup-minio-3 Cluster Name: some-name Pitr: Date: 2026-02-19 14:24:58 Type: date Status: Pitr Target: 2026-02-19T14:24:58 State: waiting Events: + cat /tmp/tmp.XDquXbwIc6 + rm /tmp/tmp.REHQx5s1KR /tmp/tmp.XDquXbwIc6 + return 0 + echo 'Restore object restore-backup-minio-3 is in waiting state after 15 minutes.' Restore object restore-backup-minio-3 is in waiting state after 15 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1