Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/logs/balancer.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ main
+ create_infra balancer-31451
+ local ns=balancer-31451
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z4E0Jlsdpk
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZsyK2aZODf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z4E0Jlsdpk
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.ZsyK2aZODf
+ rm /tmp/tmp.Z4E0Jlsdpk /tmp/tmp.ZsyK2aZODf
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n balancer-7370 backup-minio-0 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n balancer-7370 backup-minio-1 --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-minio-1" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.6yULxouctH
++ mktemp
+ local LAST_ERR=/tmp/tmp.xd11VMFLMo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6yULxouctH
+ cat /tmp/tmp.xd11VMFLMo
+ rm /tmp/tmp.6yULxouctH /tmp/tmp.xd11VMFLMo
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.D0Xzelf18y
++ mktemp
+ local LAST_ERR=/tmp/tmp.Aa46Rq4b5t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D0Xzelf18y
+ cat /tmp/tmp.Aa46Rq4b5t
+ rm /tmp/tmp.D0Xzelf18y /tmp/tmp.Aa46Rq4b5t
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ic9tjzPYbc
++ mktemp
+ local LAST_ERR=/tmp/tmp.FvdHRAQFLc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ic9tjzPYbc
+ cat /tmp/tmp.FvdHRAQFLc
+ rm /tmp/tmp.Ic9tjzPYbc /tmp/tmp.FvdHRAQFLc
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.hFD61FtLZH
++ mktemp
+ local LAST_ERR=/tmp/tmp.0NcDfH8LRt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hFD61FtLZH
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.0NcDfH8LRt
+ rm /tmp/tmp.hFD61FtLZH /tmp/tmp.0NcDfH8LRt
+ return 0
+ check_crd_for_deletion PR-1933-eb768cc5
+ local git_tag=PR-1933-eb768cc5
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1933-eb768cc5/deploy/crd.yaml
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eXJBtm7Mpp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zjfhvCBHcS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eXJBtm7Mpp
++ cat /tmp/tmp.zjfhvCBHcS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eXJBtm7Mpp
++ cat /tmp/tmp.zjfhvCBHcS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.eXJBtm7Mpp
++ cat /tmp/tmp.zjfhvCBHcS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.eXJBtm7Mpp
++ cat /tmp/tmp.zjfhvCBHcS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.eXJBtm7Mpp /tmp/tmp.zjfhvCBHcS
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ tail -n1
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.62632dLzCw
++ mktemp
+ local LAST_OUT=/tmp/tmp.G94po7AhWN
++ mktemp
+ local LAST_ERR=/tmp/tmp.gGYY27zBeP
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.oGE96hk9yX
++ seq 0 2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.62632dLzCw
+ cat /tmp/tmp.oGE96hk9yX
+ rm /tmp/tmp.62632dLzCw /tmp/tmp.oGE96hk9yX
+ return 0
namespace "balancer-7370" deleted
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G94po7AhWN
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.gGYY27zBeP
+ rm /tmp/tmp.G94po7AhWN /tmp/tmp.gGYY27zBeP
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.QCJIMCNJpC
++ mktemp
+ local LAST_ERR=/tmp/tmp.I0WYsej9Qa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QCJIMCNJpC
+ cat /tmp/tmp.I0WYsej9Qa
+ rm /tmp/tmp.QCJIMCNJpC /tmp/tmp.I0WYsej9Qa
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.8b3D3IgVgH
++ mktemp
+ local LAST_ERR=/tmp/tmp.OlxOXa3yPs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8b3D3IgVgH
namespace/psmdb-operator created
+ cat /tmp/tmp.OlxOXa3yPs
+ rm /tmp/tmp.8b3D3IgVgH /tmp/tmp.OlxOXa3yPs
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A4LJDIpijL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4EULgXwsjg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A4LJDIpijL
++ cat /tmp/tmp.4EULgXwsjg
++ rm /tmp/tmp.A4LJDIpijL /tmp/tmp.4EULgXwsjg
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6dY18G7tpk
++ mktemp
+ local LAST_ERR=/tmp/tmp.QW5glNVryg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6dY18G7tpk
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2" modified.
+ cat /tmp/tmp.QW5glNVryg
+ rm /tmp/tmp.6dY18G7tpk /tmp/tmp.QW5glNVryg
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UA5hRhdhuG
++ mktemp
+ local LAST_ERR=/tmp/tmp.SVK1pmMFDq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UA5hRhdhuG
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.SVK1pmMFDq
+ rm /tmp/tmp.UA5hRhdhuG /tmp/tmp.SVK1pmMFDq
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.AYAVmIavTH
++ mktemp
+ local LAST_ERR=/tmp/tmp.b8cF5DIhJ3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AYAVmIavTH
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.b8cF5DIhJ3
+ rm /tmp/tmp.AYAVmIavTH /tmp/tmp.b8cF5DIhJ3
+ return 0
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Cki80Nx1mY
++ mktemp
+ local LAST_ERR=/tmp/tmp.9GPeSVNA23
+ local exit_status=0
+ local timeout=4
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1933-eb768cc5") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/deploy/cw-operator.yaml
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cki80Nx1mY
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.9GPeSVNA23
+ rm /tmp/tmp.Cki80Nx1mY /tmp/tmp.9GPeSVNA23
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9R6xMrDk1d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QijIHcdGrW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9R6xMrDk1d
++ cat /tmp/tmp.QijIHcdGrW
++ rm /tmp/tmp.9R6xMrDk1d /tmp/tmp.QijIHcdGrW
++ return 0
+ wait_pod percona-server-mongodb-operator-644dbdf57f-z6grx
+ local pod=percona-server-mongodb-operator-644dbdf57f-z6grx
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-644dbdf57f-z6grx to be ready..OK
+ create_namespace balancer-31451
+ local namespace=balancer-31451
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrolebinding
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces balancer-31451'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces balancer-31451
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace balancer-31451 --ignore-not-found
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.aJl3EThkrB
++ mktemp
+ local LAST_ERR=/tmp/tmp.KeBxHEQrEv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace balancer-31451 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.sRMxD3dYOR
++ mktemp
+ local LAST_ERR=/tmp/tmp.H6bmJZ2VxY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sRMxD3dYOR
+ cat /tmp/tmp.H6bmJZ2VxY
+ rm /tmp/tmp.sRMxD3dYOR /tmp/tmp.H6bmJZ2VxY
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aJl3EThkrB
+ cat /tmp/tmp.KeBxHEQrEv
+ rm /tmp/tmp.aJl3EThkrB /tmp/tmp.KeBxHEQrEv
+ return 0
+ kubectl_bin wait --for=delete namespace balancer-31451
++ mktemp
+ local LAST_OUT=/tmp/tmp.440DrSRozf
++ mktemp
+ local LAST_ERR=/tmp/tmp.QbuwJfq9kV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace balancer-31451
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.440DrSRozf
+ cat /tmp/tmp.QbuwJfq9kV
+ rm /tmp/tmp.440DrSRozf /tmp/tmp.QbuwJfq9kV
+ return 0
+ desc 'create namespace balancer-31451'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace balancer-31451
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace balancer-31451
++ mktemp
+ local LAST_OUT=/tmp/tmp.OYdAcpjMiO
++ mktemp
+ local LAST_ERR=/tmp/tmp.HzmYX3GgOC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace balancer-31451
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OYdAcpjMiO
namespace/balancer-31451 created
+ cat /tmp/tmp.HzmYX3GgOC
+ rm /tmp/tmp.OYdAcpjMiO /tmp/tmp.HzmYX3GgOC
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EfcAQa7WmJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dhb7dHM1Rl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EfcAQa7WmJ
++ cat /tmp/tmp.dhb7dHM1Rl
++ rm /tmp/tmp.EfcAQa7WmJ /tmp/tmp.dhb7dHM1Rl
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2 --namespace=balancer-31451
++ mktemp
+ local LAST_OUT=/tmp/tmp.hYKWec3GBw
++ mktemp
+ local LAST_ERR=/tmp/tmp.D3nwlTM7CT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2 --namespace=balancer-31451
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hYKWec3GBw
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1933-eb768cc5-1-cluster2" modified.
+ cat /tmp/tmp.D3nwlTM7CT
+ rm /tmp/tmp.hYKWec3GBw /tmp/tmp.D3nwlTM7CT
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon May 19 01:47:48 2025
NAMESPACE: balancer-31451
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.balancer-31451.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace balancer-31451 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace balancer-31451

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace balancer-31451 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace balancer-31451 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xEXsq7vA5q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fk951DxYya
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xEXsq7vA5q
++ cat /tmp/tmp.Fk951DxYya
++ rm /tmp/tmp.xEXsq7vA5q /tmp/tmp.Fk951DxYya
++ return 0
+ MINIO_POD=minio-service-8967c7f7f-r5xst
+ wait_pod minio-service-8967c7f7f-r5xst
+ local pod=minio-service-8967c7f7f-r5xst
+ set +o xtrace
waiting for pod/minio-service-8967c7f7f-r5xst to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.balancer-31451.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.zmIokTMosD
++ mktemp
+ local LAST_ERR=/tmp/tmp.d0Q2CM5TwQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.balancer-31451.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zmIokTMosD
service/minio-service created
+ cat /tmp/tmp.d0Q2CM5TwQ
+ rm /tmp/tmp.zmIokTMosD /tmp/tmp.d0Q2CM5TwQ
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1CMo9rLVkV
++ mktemp
+ local LAST_ERR=/tmp/tmp.10u3oOyiwm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1CMo9rLVkV
pod "aws-cli" deleted
+ cat /tmp/tmp.10u3oOyiwm
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.1CMo9rLVkV /tmp/tmp.10u3oOyiwm
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.en7id52e8w
++ mktemp
+ local LAST_ERR=/tmp/tmp.wh43hCb9TM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.en7id52e8w
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.wh43hCb9TM
+ rm /tmp/tmp.en7id52e8w /tmp/tmp.wh43hCb9TM
+ return 0
+ log 'create PSMDB cluster'
+ set +o xtrace
[2025-05-19T01:48:42+0000] create PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/client-70.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y9Y6q9Vhbk
++ mktemp
+ local LAST_ERR=/tmp/tmp.By2sjYzbs4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/conf/client-70.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y9Y6q9Vhbk
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.By2sjYzbs4
+ rm /tmp/tmp.Y9Y6q9Vhbk /tmp/tmp.By2sjYzbs4
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/some-name-rs0.yml
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1933-eb768cc5"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.S2dussG0Xh
++ mktemp
+ local LAST_ERR=/tmp/tmp.1KjDVA7m2K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S2dussG0Xh
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.1KjDVA7m2K
+ rm /tmp/tmp.S2dussG0Xh /tmp/tmp.1KjDVA7m2K
+ return 0
+ log 'check if cfg pods started'
+ set +o xtrace
[2025-05-19T01:48:48+0000] check if cfg pods started
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready....OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kEYVkfFGOq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZeBWxeimBi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kEYVkfFGOq
++ cat /tmp/tmp.ZeBWxeimBi
++ rm /tmp/tmp.kEYVkfFGOq /tmp/tmp.ZeBWxeimBi
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jxIqE5gStB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xZ6cBEMcrE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jxIqE5gStB
++ cat /tmp/tmp.xZ6cBEMcrE
++ rm /tmp/tmp.jxIqE5gStB /tmp/tmp.xZ6cBEMcrE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ log 'check if all shards started'
+ set +o xtrace
[2025-05-19T01:50:14+0000] check if all shards started
+ wait_for_running some-name-rs0 3 false
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JwyJqlIYJt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kf9JKDQQNu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JwyJqlIYJt
++ cat /tmp/tmp.Kf9JKDQQNu
++ rm /tmp/tmp.JwyJqlIYJt /tmp/tmp.Kf9JKDQQNu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vua867MPDM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fiMvqq2FRI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vua867MPDM
++ cat /tmp/tmp.fiMvqq2FRI
++ rm /tmp/tmp.vua867MPDM /tmp/tmp.fiMvqq2FRI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-rs1 3 false
+ local name=some-name-rs1
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs1
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs1-0
+ local pod=some-name-rs1-0
+ set +o xtrace
waiting for pod/some-name-rs1-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs1-1
+ local pod=some-name-rs1-1
+ set +o xtrace
waiting for pod/some-name-rs1-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oCkLAWKN1T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oDFIyGhAi5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oCkLAWKN1T
++ cat /tmp/tmp.oDFIyGhAi5
++ rm /tmp/tmp.oCkLAWKN1T /tmp/tmp.oDFIyGhAi5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs1-2
+ local pod=some-name-rs1-2
+ set +o xtrace
waiting for pod/some-name-rs1-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6HFrB45ZNc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VOfKxNHEg4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6HFrB45ZNc
++ cat /tmp/tmp.VOfKxNHEg4
++ rm /tmp/tmp.6HFrB45ZNc /tmp/tmp.VOfKxNHEg4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-rs2 3 false
+ local name=some-name-rs2
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs2
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs2-0
+ local pod=some-name-rs2-0
+ set +o xtrace
waiting for pod/some-name-rs2-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs2-1
+ local pod=some-name-rs2-1
+ set +o xtrace
waiting for pod/some-name-rs2-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RjyWdjLVz5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UTdTcoaS3z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RjyWdjLVz5
++ cat /tmp/tmp.UTdTcoaS3z
++ rm /tmp/tmp.RjyWdjLVz5 /tmp/tmp.UTdTcoaS3z
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs2-2
+ local pod=some-name-rs2-2
+ set +o xtrace
waiting for pod/some-name-rs2-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xEaTUDqW6M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q0NjQaQD3m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xEaTUDqW6M
++ cat /tmp/tmp.q0NjQaQD3m
++ rm /tmp/tmp.xEaTUDqW6M /tmp/tmp.q0NjQaQD3m
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ log 'check if mongos pods started'
+ set +o xtrace
[2025-05-19T01:51:14+0000] check if mongos pods started
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready....OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.....OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rHTbbx30EM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6mnWRgXQ4z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rHTbbx30EM
++ cat /tmp/tmp.6mnWRgXQ4z
++ rm /tmp/tmp.rHTbbx30EM /tmp/tmp.6mnWRgXQ4z
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kBDdm9pcaT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jB8wEwckG2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kBDdm9pcaT
++ cat /tmp/tmp.jB8wEwckG2
++ rm /tmp/tmp.kBDdm9pcaT /tmp/tmp.jB8wEwckG2
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ check_balancer some-name true 10
+ local cluster=some-name
+ local expected=true
+ local delay=10
+ local balancer_running
+ log 'sleeping for 10 seconds...'
+ set +o xtrace
[2025-05-19T01:52:07+0000] sleeping for 10 seconds...
+ sleep 10
++ grep -E -v 'Warning|cfg'
++ grep -E 'true|false'
++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local 'command=sh.getBalancerState()'
++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.crOJjzcfZl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8j6SIo9cjH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.crOJjzcfZl
+++ cat /tmp/tmp.8j6SIo9cjH
+++ rm /tmp/tmp.crOJjzcfZl /tmp/tmp.8j6SIo9cjH
+++ return 0
++ local client_container=psmdb-client-65ff95489b-qkppc
++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451 == *cfg* ]]
++ replica_set=cfg
++ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nFmwymGSJu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HXrqUOBtrN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nFmwymGSJu
++ cat /tmp/tmp.HXrqUOBtrN
++ rm /tmp/tmp.nFmwymGSJu /tmp/tmp.HXrqUOBtrN
++ return 0
+ balancer_running=true
+ echo -n 'checking if balancer status is true...'
checking if balancer status is true...+ [[ true != \t\r\u\e ]]
+ echo OK
OK
+ write_data some-name
+ local cluster=some-name
+ log 'create user'
+ set +o xtrace
[2025-05-19T01:52:22+0000] create user
+ run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.balancer-31451 mongodb .svc.cluster.local --quiet 27017 mongosh
+ local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.balancer-31451
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=--quiet
+ local port=27017
+ local mongo_bin=mongosh
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yyYFp8pb0D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wgYEOcqEqt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yyYFp8pb0D
++ cat /tmp/tmp.wgYEOcqEqt
++ rm /tmp/tmp.yyYFp8pb0D /tmp/tmp.wgYEOcqEqt
++ return 0
+ local client_container=psmdb-client-65ff95489b-qkppc
+ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongosh mongodb://userAdmin:userAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LTHP763eQ2
++ mktemp
+ local LAST_ERR=/tmp/tmp.dI65ONcOlt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongosh mongodb://userAdmin:userAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LTHP763eQ2
[direct: mongos] admin> {
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1747619546, i: 4 }),
    signature: {
      hash: Binary.createFromBase64('mLt5AmD/xP8RwUEtgxUdmIfPhdY=', 0),
      keyId: Long('7505968177445077004')
    }
  },
  operationTime: Timestamp({ t: 1747619546, i: 4 })
}
[direct: mongos] admin> + cat /tmp/tmp.dI65ONcOlt
+ rm /tmp/tmp.LTHP763eQ2 /tmp/tmp.dI65ONcOlt
+ return 0
+ sleep 2
+ log 'enable sharding'
+ set +o xtrace
[2025-05-19T01:52:28+0000] enable sharding
+ run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451 mongodb .svc.cluster.local --quiet 27017 mongosh
+ local 'command=sh.enableSharding("app")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=--quiet
+ local port=27017
+ local mongo_bin=mongosh
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UlDVDJx8Mg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FWcbHeHuBN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UlDVDJx8Mg
++ cat /tmp/tmp.FWcbHeHuBN
++ rm /tmp/tmp.UlDVDJx8Mg /tmp/tmp.FWcbHeHuBN
++ return 0
+ local client_container=psmdb-client-65ff95489b-qkppc
+ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
++ mktemp
+ local LAST_OUT=/tmp/tmp.aB1ZWgBVNW
++ mktemp
+ local LAST_ERR=/tmp/tmp.hTtUksyrgn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aB1ZWgBVNW
[direct: mongos] admin> {
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1747619553, i: 10 }),
    signature: {
      hash: Binary.createFromBase64('gFka9ipDiJZ1TL8vMP7ghvy7bgk=', 0),
      keyId: Long('7505968177445077004')
    }
  },
  operationTime: Timestamp({ t: 1747619553, i: 7 })
}
[direct: mongos] admin> + cat /tmp/tmp.hTtUksyrgn
+ rm /tmp/tmp.aB1ZWgBVNW /tmp/tmp.hTtUksyrgn
+ return 0
+ sleep 2
+ log 'shard collection'
+ set +o xtrace
[2025-05-19T01:52:36+0000] shard collection
+ run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451 mongodb .svc.cluster.local --quiet 27017 mongosh
+ local 'command=sh.shardCollection("app.city", { _id: 1 } )'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=--quiet
+ local port=27017
+ local mongo_bin=mongosh
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qM6htJIp56
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nedy1YdiRv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qM6htJIp56
++ cat /tmp/tmp.nedy1YdiRv
++ rm /tmp/tmp.qM6htJIp56 /tmp/tmp.nedy1YdiRv
++ return 0
+ local client_container=psmdb-client-65ff95489b-qkppc
+ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LaG6KrNEws
++ mktemp
+ local LAST_ERR=/tmp/tmp.4kC6lLM0DW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-31451.svc.cluster.local:27017/admin --quiet'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LaG6KrNEws
[direct: mongos] admin> {
  collectionsharded: 'app.city',
  ok: 1,
  '$clusterTime': {
    clusterTime: Timestamp({ t: 1747619560, i: 38 }),
    signature: {
      hash: Binary.createFromBase64('0489vBV0CdWvJcf2PPBOw9dggP8=', 0),
      keyId: Long('7505968177445077004')
    }
  },
  operationTime: Timestamp({ t: 1747619560, i: 37 })
}
[direct: mongos] admin> + cat /tmp/tmp.4kC6lLM0DW
+ rm /tmp/tmp.LaG6KrNEws /tmp/tmp.4kC6lLM0DW
+ return 0
+ log 'write data (this can take some time, be patient)'
+ set +o xtrace
[2025-05-19T01:52:40+0000] write data (this can take some time, be patient)
+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/data.js user:pass@some-name-mongos.balancer-31451 mongodb .svc.cluster.local --quiet mongosh
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/data.js
+ local uri=user:pass@some-name-mongos.balancer-31451
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=--quiet
+ local mongo_bin=mongosh
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Cx1iXVn63F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EkrbB0VeDT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Cx1iXVn63F
++ cat /tmp/tmp.EkrbB0VeDT
++ rm /tmp/tmp.Cx1iXVn63F /tmp/tmp.EkrbB0VeDT
++ return 0
+ local client_container=psmdb-client-65ff95489b-qkppc
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/data.js
+ name=data.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/data.js balancer-31451/psmdb-client-65ff95489b-qkppc:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.CLqEPloFHc
++ mktemp
+ local LAST_ERR=/tmp/tmp.VlDkhIZ0q8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/data.js balancer-31451/psmdb-client-65ff95489b-qkppc:/tmp
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CLqEPloFHc
+ cat /tmp/tmp.VlDkhIZ0q8
+ rm /tmp/tmp.CLqEPloFHc /tmp/tmp.VlDkhIZ0q8
+ return 0
+ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'mongosh mongodb://user:pass@some-name-mongos.balancer-31451.svc.cluster.local/admin --quiet /tmp/data.js'
++ mktemp
+ local LAST_OUT=/tmp/tmp.L29Uq2ShCb
++ mktemp
+ local LAST_ERR=/tmp/tmp.iH5Z6QEu03
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'mongosh mongodb://user:pass@some-name-mongos.balancer-31451.svc.cluster.local/admin --quiet /tmp/data.js'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L29Uq2ShCb
+ cat /tmp/tmp.iH5Z6QEu03
+ rm /tmp/tmp.L29Uq2ShCb /tmp/tmp.iH5Z6QEu03
+ return 0
+ check_backup_and_restore some-name 0 true
+ local cluster=some-name
+ local backup_suffix=0
+ local balancer_end_state=true
+ local backup_name=backup-minio-0
+ log 'running backup: backup-minio-0'
+ set +o xtrace
[2025-05-19T01:54:43+0000] running backup: backup-minio-0
+ run_backup minio backup-minio-0
+ local storage=minio
+ local backup_name=backup-minio-0
+ local type=logical
+ desc 'run backup backup-minio-0'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-0
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-minio-0"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.q8hrA3V6HL
++ mktemp
+ local LAST_ERR=/tmp/tmp.0GlUAdRifH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q8hrA3V6HL
perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created
+ cat /tmp/tmp.0GlUAdRifH
+ rm /tmp/tmp.q8hrA3V6HL /tmp/tmp.0GlUAdRifH
+ return 0
+ wait_backup backup-minio-0 requested
+ local backup_name=backup-minio-0
+ local target_state=requested
+ set +o xtrace
waiting for backup-minio-0 to reach requested state.
+ log 'checking if balancer is disabled'
+ set +o xtrace
[2025-05-19T01:54:49+0000] checking if balancer is disabled
+ check_balancer some-name false
+ local cluster=some-name
+ local expected=false
+ local delay=0
+ local balancer_running
+ log 'sleeping for 0 seconds...'
+ set +o xtrace
[2025-05-19T01:54:49+0000] sleeping for 0 seconds...
+ sleep 0
++ grep -E -v 'Warning|cfg'
++ grep -E 'true|false'
++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local 'command=sh.getBalancerState()'
++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local driver=mongodb+srv
++ local suffix=.svc.cluster.local
++ local mongo_flag=
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9reJS2lonl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8lY5Qu22o1
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.9reJS2lonl
+++ cat /tmp/tmp.8lY5Qu22o1
+++ rm /tmp/tmp.9reJS2lonl /tmp/tmp.8lY5Qu22o1
+++ return 0
++ local client_container=psmdb-client-65ff95489b-qkppc
++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451 == *cfg* ]]
++ replica_set=cfg
++ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nyWVOmyo22
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8rsjf8Cxkf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nyWVOmyo22
++ cat /tmp/tmp.8rsjf8Cxkf
++ rm /tmp/tmp.nyWVOmyo22 /tmp/tmp.8rsjf8Cxkf
++ return 0
+ balancer_running=false
+ echo -n 'checking if balancer status is false...'
checking if balancer status is false...+ [[ false != \f\a\l\s\e ]]
+ echo OK
OK
+ wait_backup backup-minio-0 ready
+ local backup_name=backup-minio-0
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-0 to reach ready state.........
+ log 'checking if balancer is true after backup'
+ set +o xtrace
[2025-05-19T01:55:20+0000] checking if balancer is true after backup
+ check_balancer some-name true 10
+ local cluster=some-name
+ local expected=true
+ local delay=10
+ local balancer_running
+ log 'sleeping for 10 seconds...'
+ set +o xtrace
[2025-05-19T01:55:20+0000] sleeping for 10 seconds...
+ sleep 10
++ grep -E -v 'Warning|cfg'
++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local 'command=sh.getBalancerState()'
++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451
++ local driver=mongodb+srv
++ grep -E 'true|false'
++ local suffix=.svc.cluster.local
++ local mongo_flag=
+++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.asOjVwq8Tj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rxNV6YhePT
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.asOjVwq8Tj
+++ cat /tmp/tmp.rxNV6YhePT
+++ rm /tmp/tmp.asOjVwq8Tj /tmp/tmp.rxNV6YhePT
+++ return 0
++ local client_container=psmdb-client-65ff95489b-qkppc
++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451 == *cfg* ]]
++ replica_set=cfg
++ kubectl_bin exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NKgYI0zdoa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qVfwVKsQPN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec psmdb-client-65ff95489b-qkppc -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-31451.svc.cluster.local/admin?ssl=false\&replicaSet=cfg '
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NKgYI0zdoa
++ cat /tmp/tmp.qVfwVKsQPN
++ rm /tmp/tmp.NKgYI0zdoa /tmp/tmp.qVfwVKsQPN
++ return 0
+ balancer_running=true
+ echo -n 'checking if balancer status is true...'
checking if balancer status is true...+ [[ true != \t\r\u\e ]]
+ echo OK
OK
+ log 'running restore: restore-backup-minio-0'
+ set +o xtrace
[2025-05-19T01:55:34+0000] running restore: restore-backup-minio-0
+ run_restore backup-minio-0
+ local backup_name=backup-minio-0
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-0/'
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-0/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1933/e2e-tests/balancer/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PnkCzOZZ9j
++ mktemp
+ local LAST_ERR=/tmp/tmp.JejZHEcUra
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PnkCzOZZ9j
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created
+ cat /tmp/tmp.JejZHEcUra
+ rm /tmp/tmp.PnkCzOZZ9j /tmp/tmp.JejZHEcUra
+ return 0
+ wait_restore backup-minio-0 some-name requested 0
+ local backup_name=backup-minio-0
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-backup-minio-0 to reach requested state....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2025-05-19T01:55:03.073Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "650e65f8-8cd2-428b-948b-001f02637ea9", "currentJob": {"Name":"backup-minio-0","Type":0}}
2025-05-19T01:55:03.164Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "650e65f8-8cd2-428b-948b-001f02637ea9"}
2025-05-19T01:55:03.166Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "650e65f8-8cd2-428b-948b-001f02637ea9", "backup": "backup-minio-0", "pbmName": "2025-05-19T01:54:47Z", "meta": {"type":"logical","opid":"682a8f6779c4a3a4073a3816","name":"2025-05-19T01:54:47Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs2","backup_name":"2025-05-19T01:54:47Z/rs2/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs2/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619693,"first_write_ts":{"T":1747619687,"I":1},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619693,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs0","backup_name":"2025-05-19T01:54:47Z/rs0/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs0/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619692,"first_write_ts":{"T":1747619680,"I":11},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619692,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs1","backup_name":"2025-05-19T01:54:47Z/rs1/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs1/oplog","start_ts":1747619688,"status":"running","last_transition_ts":1747619688,"first_write_ts":{"T":1747619682,"I":42},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"cfg","backup_name":"2025-05-19T01:54:47Z/cfg/metadata.json","oplog_name":"2025-05-19T01:54:47Z/cfg/oplog","start_ts":1747619688,"status":"dumpDone","iscs":true,"last_transition_ts":1747619691,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":0,"I":0},"node":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619691,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747619688,"last_transition_ts":1747619691,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747619698,"I":6},"status":"running","conditions":[{"timestamp":1747619688,"status":"starting"},{"timestamp":1747619691,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-31451.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.balancer-31451.svc.cluster.local:27017","some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}}
2025-05-19T01:55:03.171Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "650e65f8-8cd2-428b-948b-001f02637ea9"}
2025-05-19T01:55:08.172Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10"}
2025-05-19T01:55:08.270Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:08.270Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10", "currentJob": {"Name":"backup-minio-0","Type":0}}
2025-05-19T01:55:08.414Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10"}
2025-05-19T01:55:08.416Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10", "backup": "backup-minio-0", "pbmName": "2025-05-19T01:54:47Z", "meta": {"type":"logical","opid":"682a8f6779c4a3a4073a3816","name":"2025-05-19T01:54:47Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs2","backup_name":"2025-05-19T01:54:47Z/rs2/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs2/oplog","start_ts":1747619688,"status":"done","last_transition_ts":1747619707,"first_write_ts":{"T":1747619687,"I":1},"last_write_ts":{"T":1747619692,"I":35},"node":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619693,"status":"dumpDone"},{"timestamp":1747619707,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs0","backup_name":"2025-05-19T01:54:47Z/rs0/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs0/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619692,"first_write_ts":{"T":1747619680,"I":11},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619692,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs1","backup_name":"2025-05-19T01:54:47Z/rs1/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs1/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619705,"first_write_ts":{"T":1747619682,"I":42},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619705,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"cfg","backup_name":"2025-05-19T01:54:47Z/cfg/metadata.json","oplog_name":"2025-05-19T01:54:47Z/cfg/oplog","start_ts":1747619688,"status":"dumpDone","iscs":true,"last_transition_ts":1747619691,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1747619706,"I":1},"node":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619691,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":11700,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747619688,"last_transition_ts":1747619706,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747619703,"I":7},"status":"dumpDone","conditions":[{"timestamp":1747619688,"status":"starting"},{"timestamp":1747619691,"status":"running"},{"timestamp":1747619706,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-31451.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.balancer-31451.svc.cluster.local:27017","some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}}
2025-05-19T01:55:08.423Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "ce0737cc-c8ec-42d6-b07e-8ab37c176b10"}
2025-05-19T01:55:13.426Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142"}
2025-05-19T01:55:13.518Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:13.518Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142", "currentJob": {"Name":"backup-minio-0","Type":0}}
2025-05-19T01:55:13.635Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142"}
2025-05-19T01:55:13.637Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142", "backup": "backup-minio-0", "pbmName": "2025-05-19T01:54:47Z", "meta": {"type":"logical","opid":"682a8f6779c4a3a4073a3816","name":"2025-05-19T01:54:47Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs2","backup_name":"2025-05-19T01:54:47Z/rs2/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs2/oplog","start_ts":1747619688,"status":"done","last_transition_ts":1747619707,"first_write_ts":{"T":1747619687,"I":1},"last_write_ts":{"T":1747619692,"I":35},"node":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619693,"status":"dumpDone"},{"timestamp":1747619707,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs0","backup_name":"2025-05-19T01:54:47Z/rs0/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs0/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619692,"first_write_ts":{"T":1747619680,"I":11},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619692,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs1","backup_name":"2025-05-19T01:54:47Z/rs1/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs1/oplog","start_ts":1747619688,"status":"dumpDone","last_transition_ts":1747619705,"first_write_ts":{"T":1747619682,"I":42},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619705,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"cfg","backup_name":"2025-05-19T01:54:47Z/cfg/metadata.json","oplog_name":"2025-05-19T01:54:47Z/cfg/oplog","start_ts":1747619688,"status":"dumpDone","iscs":true,"last_transition_ts":1747619691,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1747619706,"I":1},"node":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619691,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":11700,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747619688,"last_transition_ts":1747619706,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747619708,"I":7},"status":"dumpDone","conditions":[{"timestamp":1747619688,"status":"starting"},{"timestamp":1747619691,"status":"running"},{"timestamp":1747619706,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-31451.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.balancer-31451.svc.cluster.local:27017","some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}}
2025-05-19T01:55:13.642Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "fe51a22e-5ddf-415e-a102-93ad67aae142"}
2025-05-19T01:55:18.643Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265"}
2025-05-19T01:55:18.724Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:18.724Z	DEBUG	Checking for active jobs	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265", "currentJob": {"Name":"backup-minio-0","Type":0}}
2025-05-19T01:55:18.810Z	INFO	Acquiring the backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265"}
2025-05-19T01:55:18.811Z	DEBUG	backupStatus	Got backup meta	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265", "backup": "backup-minio-0", "pbmName": "2025-05-19T01:54:47Z", "meta": {"type":"logical","opid":"682a8f6779c4a3a4073a3816","name":"2025-05-19T01:54:47Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs2","backup_name":"2025-05-19T01:54:47Z/rs2/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs2/oplog","start_ts":1747619688,"status":"done","last_transition_ts":1747619707,"first_write_ts":{"T":1747619687,"I":1},"last_write_ts":{"T":1747619692,"I":35},"node":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619693,"status":"dumpDone"},{"timestamp":1747619707,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs0","backup_name":"2025-05-19T01:54:47Z/rs0/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs0/oplog","start_ts":1747619688,"status":"done","last_transition_ts":1747619714,"first_write_ts":{"T":1747619680,"I":11},"last_write_ts":{"T":1747619704,"I":10},"node":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619692,"status":"dumpDone"},{"timestamp":1747619714,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"rs1","backup_name":"2025-05-19T01:54:47Z/rs1/metadata.json","oplog_name":"2025-05-19T01:54:47Z/rs1/oplog","start_ts":1747619688,"status":"done","last_transition_ts":1747619715,"first_write_ts":{"T":1747619682,"I":42},"last_write_ts":{"T":1747619705,"I":6},"node":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619705,"status":"dumpDone"},{"timestamp":1747619715,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"},{"name":"cfg","backup_name":"2025-05-19T01:54:47Z/cfg/metadata.json","oplog_name":"2025-05-19T01:54:47Z/cfg/oplog","start_ts":1747619688,"status":"done","iscs":true,"last_transition_ts":1747619716,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1747619706,"I":1},"node":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","conditions":[{"timestamp":1747619688,"status":"running"},{"timestamp":1747619691,"status":"dumpDone"},{"timestamp":1747619716,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"8.0.8-3"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":25425713,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747619688,"last_transition_ts":1747619717,"first_write_ts":{"T":1747619688,"I":56},"last_write_ts":{"T":1747619692,"I":35},"hb":{"T":1747619713,"I":7},"status":"done","conditions":[{"timestamp":1747619688,"status":"starting"},{"timestamp":1747619691,"status":"running"},{"timestamp":1747619706,"status":"dumpDone"},{"timestamp":1747619717,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-31451.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-31451.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-31451.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017","some-name-rs2-1.some-name-rs2.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-31451.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-1.some-name-rs1.balancer-31451.svc.cluster.local:27017","some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"],"ack":"some-name-rs1-2.some-name-rs1.balancer-31451.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}}
2025-05-19T01:55:18.814Z	INFO	Backup state changed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265", "previous": "running", "current": "ready"}
2025-05-19T01:55:18.859Z	INFO	Releasing backup lock	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265", "lease": "psmdb-some-name-backup-lock"}
2025-05-19T01:55:18.876Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d19d8e24-abcb-4679-8f40-121ddedbb265"}
2025-05-19T01:55:18.876Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "e6d67120-427a-471e-b0aa-47acc05064fa"}
2025-05-19T01:55:18.876Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "e6d67120-427a-471e-b0aa-47acc05064fa"}
2025-05-19T01:55:23.876Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d530bf44-2903-43ad-9283-5196f2c6b2bd"}
2025-05-19T01:55:23.876Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "backup-minio-0", "reconcileID": "d530bf44-2903-43ad-9283-5196f2c6b2bd"}
2025-05-19T01:55:38.168Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "1cb473c8-25ab-4361-abdb-12dd3a3e3a19", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:38.308Z	INFO	Terminating mongos pods	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "1cb473c8-25ab-4361-abdb-12dd3a3e3a19"}
2025-05-19T01:55:43.072Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "712f0ac9-1ba7-4e91-ac91-14ffe11ee163", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:55:43.072Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "712f0ac9-1ba7-4e91-ac91-14ffe11ee163", "previous": "ready", "current": "initializing"}
2025-05-19T01:55:43.324Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "1cf19e39-deb7-4656-b855-02407af83514", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:43.495Z	INFO	Waiting for mongos pods to terminate	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "1cf19e39-deb7-4656-b855-02407af83514"}
2025-05-19T01:55:46.014Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "a09be193-8a91-4d5d-93eb-e558d44168cf", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:55:48.496Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "19a6b922-9961-4ba7-a6ff-03aef40cfe4f", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:48.642Z	INFO	Waiting for mongos pods to terminate	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "19a6b922-9961-4ba7-a6ff-03aef40cfe4f"}
2025-05-19T01:55:51.474Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "667690b9-c581-4e86-9dbf-49e2c7e306e6", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:55:53.643Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "38e806cc-0e12-4a3a-92b4-abdadbffb326", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:53.817Z	INFO	Waiting for mongos pods to terminate	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "38e806cc-0e12-4a3a-92b4-abdadbffb326"}
2025-05-19T01:55:58.818Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "caabc97a-e90b-47b3-a658-f3fedab633ef", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:58.883Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "f0b964a1-2bd6-4ed4-88e7-509d4659d968", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:55:59.041Z	INFO	Starting restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "caabc97a-e90b-47b3-a658-f3fedab633ef", "backup": "2025-05-19T01:54:47Z"}
2025-05-19T01:55:59.041Z	INFO	Starting logical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "caabc97a-e90b-47b3-a658-f3fedab633ef", "backup": "2025-05-19T01:54:47Z"}
2025-05-19T01:55:59.041Z	INFO	Sending restore command	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "caabc97a-e90b-47b3-a658-f3fedab633ef", "restoreCmd": "name: 2025-05-19T01:55:59.041768116Z, snapshot: 2025-05-19T01:54:47Z"}
2025-05-19T01:55:59.046Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "caabc97a-e90b-47b3-a658-f3fedab633ef", "previous": "", "current": "requested"}
2025-05-19T01:55:59.820Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "76ed0a5f-c49c-4524-b002-e9af7ff9a414", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:55:59.911Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "76ed0a5f-c49c-4524-b002-e9af7ff9a414", "previous": "requested", "current": "running"}
2025-05-19T01:56:00.509Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "b9fc0047-949c-41cc-b3f8-4bca6ff71697", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:04.821Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "ede9421a-8be3-4054-8601-37bd05eeafda", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:06.977Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "3f2c0744-21bc-4744-8faa-07633e2ff8d7", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:09.950Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "ea365694-00f6-4f71-9075-99321a50ab08", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:15.066Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "71e16897-68ae-4ef6-943b-b02b382967b2", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:15.914Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "af25c633-d962-4b19-b093-c843ec2038bb", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:20.172Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "62809798-7f4f-4aed-9529-eab77727c688", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:24.666Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "a2726d4b-0732-4985-a57f-1bba2e585ca3", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:25.267Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "698ca668-69d2-4bfa-a8d8-f45fa79158aa", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:30.407Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "2ada12ab-1b89-425f-bcbd-f6feb7eb41f3", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:35.038Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "d4bbf5e6-a9dc-4ed2-89af-8be9f5e87e6a", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:35.506Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "2aeeebb4-d33c-4bf2-8263-f29f1f00a095", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:40.887Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "66b1d493-a23a-4a2b-96d3-251dc2c08646", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:43.474Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "29cbdf15-ff0b-44cd-94da-516c372a223e", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:45.981Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "b174e37c-c8ee-4ecc-806c-0b33745eb99f", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:51.081Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "dc320b71-91ef-4d1f-9517-af631f0494b6", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:56:51.722Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "b9167082-b416-4d63-b457-3baecfb69d79", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:56:56.182Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "899d8934-3a46-42eb-99a9-a713bbed3054", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:01.276Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "d2944d11-ce89-4420-a3d7-f5abb60a9bdd", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:01.399Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "3544d241-f247-44d5-9e27-58a6e5e2f552", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:06.402Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "02595a88-0b22-44ef-bcf1-fe13c83eb8c4", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:09.634Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "e13ad48a-20fa-4286-b076-b1382295bcb1", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:11.506Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "a24f50ee-0e41-4c4b-8d88-ae5877e97129", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:16.623Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "1a1f9147-1836-4103-ad61-bf60f3072267", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:18.717Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "4b75edb8-87f8-4a9b-a130-1791e7df9f0e", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:21.735Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "8b8fcc21-8b78-4546-a95f-262b39a9dcce", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:26.804Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "e83948ea-31b9-4dbf-b08a-bc6c507012a4", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:26.836Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "d9472875-8457-4244-97cf-8c8db31c66ac", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:31.928Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "2a4006dc-e429-4fcf-914e-062368b21db4", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:35.855Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "60857b35-5bf8-44da-816e-9fa7c3591f95", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:37.049Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "f8a6c455-9cd6-4e36-b0b5-71b32b2936ec", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:42.151Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "3c436c8a-6de6-4ef0-8404-cb726abd6f3f", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:44.010Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "e505785f-9e39-4790-8695-9b3ab22863ae", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:47.269Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "7e2135c7-5173-4b1c-88b9-5ab501b8a829", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:52.306Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "e571cab7-ca5d-4a0b-aad5-912d9d3deec5", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:57:52.366Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "70eb1d01-f1f9-44ce-99c5-7c06858a566b", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:57:57.464Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "d28e2ca7-9bd8-474b-a133-b1ed2b369c1c", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:58:01.088Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "b32e5279-fe8c-40f7-bb25-295c2474ab32", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:02.585Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "8cd8b650-65f9-4ee4-b292-952fe0594043", "cluster": "some-name", "namespace": "balancer-31451"}
2025-05-19T01:58:02.668Z	INFO	Restore state changed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-0","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "restore-backup-minio-0", "reconcileID": "8cd8b650-65f9-4ee4-b292-952fe0594043", "previous": "running", "current": "ready"}
2025-05-19T01:58:08.792Z	DEBUG	Object created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "afe1d230-de0e-470e-a66e-4e0cb88e5db0", "name": "some-name-mongos", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "generation": 0, "resourceVersion": ""}
2025-05-19T01:58:08.825Z	DEBUG	Object created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "afe1d230-de0e-470e-a66e-4e0cb88e5db0", "name": "some-name-mongos-", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "generation": 0, "resourceVersion": ""}
2025-05-19T01:58:09.400Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "afe1d230-de0e-470e-a66e-4e0cb88e5db0", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:12.750Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "88046fe3-3b47-4fb0-8dc8-2db0e5a43ac4", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:18.119Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "ade7dce3-6f76-44e1-9ac1-a346a6be274f", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:25.775Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "be1975f6-2088-4c75-b3e0-52261a5bcba8", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:28.831Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "0503710c-aaed-44b7-9a75-06a76551ec25", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:33.868Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "e74f3a96-c8b2-425f-b072-335b907bd9a3", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:37.172Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "013feef8-6ec6-426f-87cb-ef568ee1a730", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:41.382Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "0cf7a67a-8970-45e3-a8b8-ecbbcf7e5fd9", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"}
2025-05-19T01:58:49.635Z	INFO	balancer enabled	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "02642080-9706-4ac7-b45d-ecd7f75d3189"}
2025-05-19T01:58:49.897Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-31451"}, "namespace": "balancer-31451", "name": "some-name", "reconcileID": "02642080-9706-4ac7-b45d-ecd7f75d3189", "previous": "initializing", "current": "ready"}
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-minio-0","namespace":"balancer-31451"},"spec":{"backupName":"backup-minio-0","clusterName":"some-name"}}
  creationTimestamp: "2025-05-19T01:55:38Z"
  generation: 1
  name: restore-backup-minio-0
  namespace: balancer-31451
  resourceVersion: "24852"
  uid: 40d0d1d1-0697-466a-8af9-be837fa20aa5
spec:
  backupName: backup-minio-0
  clusterName: some-name
status:
  completed: "2025-05-19T01:58:00Z"
  pbmName: "2025-05-19T01:55:59.041768116Z"
  state: ready
Restore object restore-backup-minio-0 is in ready state.
something went wrong with operator or kubernetes cluster