Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/logs/balancer.log WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1 + main + create_infra balancer-19260 + local ns=balancer-19260 + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.9GStkrMPSX ++ mktemp + local LAST_ERR=/tmp/tmp.hEZsHCoeR1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9GStkrMPSX customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.hEZsHCoeR1 + rm /tmp/tmp.9GStkrMPSX /tmp/tmp.hEZsHCoeR1 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.pgLkaVgC3O ++ mktemp + local LAST_ERR=/tmp/tmp.HanKC25acI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pgLkaVgC3O + cat /tmp/tmp.HanKC25acI + rm /tmp/tmp.pgLkaVgC3O /tmp/tmp.HanKC25acI + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.NOBONgXUZk ++ mktemp + local LAST_ERR=/tmp/tmp.2AsFVFivsH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NOBONgXUZk + cat /tmp/tmp.2AsFVFivsH + rm /tmp/tmp.NOBONgXUZk /tmp/tmp.2AsFVFivsH + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.DyOEJUfn6W ++ mktemp + local LAST_ERR=/tmp/tmp.tEVE3GQN5H + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DyOEJUfn6W + cat /tmp/tmp.tEVE3GQN5H + rm /tmp/tmp.DyOEJUfn6W /tmp/tmp.tEVE3GQN5H + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1l2Bdi2M8s ++ mktemp + local LAST_ERR=/tmp/tmp.QKzXN9KAA6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1l2Bdi2M8s clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.QKzXN9KAA6 + rm /tmp/tmp.1l2Bdi2M8s /tmp/tmp.QKzXN9KAA6 + return 0 + check_crd_for_deletion PR-1872-66b64516 + local git_tag=PR-1872-66b64516 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1872-66b64516/deploy/crd.yaml ++ /usr/bin/sed s/---//g ++ yq eval .metadata.name ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RplPqimL8V +++ mktemp ++ local LAST_ERR=/tmp/tmp.DhjwAue1cA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.RplPqimL8V ++ cat /tmp/tmp.DhjwAue1cA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.RplPqimL8V ++ cat /tmp/tmp.DhjwAue1cA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.RplPqimL8V ++ cat /tmp/tmp.DhjwAue1cA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.RplPqimL8V ++ cat /tmp/tmp.DhjwAue1cA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.RplPqimL8V /tmp/tmp.DhjwAue1cA ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + awk '{print$1}' + kubectl_bin get ns + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.PoJO6ZZyuJ + local LAST_OUT=/tmp/tmp.3VizCyqoaO + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp + local LAST_ERR=/tmp/tmp.EvaoIZC2c2 + local exit_status=0 + local timeout=4 ++ mktemp ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + local LAST_ERR=/tmp/tmp.8dM0FBhmx4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PoJO6ZZyuJ + cat /tmp/tmp.8dM0FBhmx4 + rm /tmp/tmp.PoJO6ZZyuJ /tmp/tmp.8dM0FBhmx4 + return 0 namespace "balancer-31497" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3VizCyqoaO namespace "psmdb-operator" deleted + cat /tmp/tmp.EvaoIZC2c2 + rm /tmp/tmp.3VizCyqoaO /tmp/tmp.EvaoIZC2c2 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.H6gsXl9cMa ++ mktemp + local LAST_ERR=/tmp/tmp.LrLmqVi8Wv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H6gsXl9cMa + cat /tmp/tmp.LrLmqVi8Wv + rm /tmp/tmp.H6gsXl9cMa /tmp/tmp.LrLmqVi8Wv + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.kTeUSeFGJr ++ mktemp + local LAST_ERR=/tmp/tmp.rt2SqfN0tm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kTeUSeFGJr namespace/psmdb-operator created + cat /tmp/tmp.rt2SqfN0tm + rm /tmp/tmp.kTeUSeFGJr /tmp/tmp.rt2SqfN0tm + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.O0AUicqyqX +++ mktemp ++ local LAST_ERR=/tmp/tmp.8LQ4CHRnpy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.O0AUicqyqX ++ cat /tmp/tmp.8LQ4CHRnpy ++ rm /tmp/tmp.O0AUicqyqX /tmp/tmp.8LQ4CHRnpy ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.uDQAn1SiWV ++ mktemp + local LAST_ERR=/tmp/tmp.90NFK4o7Ce + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uDQAn1SiWV Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2" modified. + cat /tmp/tmp.90NFK4o7Ce + rm /tmp/tmp.uDQAn1SiWV /tmp/tmp.90NFK4o7Ce + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.8wy5HFcFG3 ++ mktemp + local LAST_ERR=/tmp/tmp.Uuo7ffhJAE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8wy5HFcFG3 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.Uuo7ffhJAE + rm /tmp/tmp.8wy5HFcFG3 /tmp/tmp.Uuo7ffhJAE + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.3pCxN6a8Ei ++ mktemp + local LAST_ERR=/tmp/tmp.pNeHRIrYDi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3pCxN6a8Ei clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.pNeHRIrYDi + rm /tmp/tmp.3pCxN6a8Ei /tmp/tmp.pNeHRIrYDi + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.MVVtTlccJ8 ++ mktemp + local LAST_ERR=/tmp/tmp.hhTxzWkrsx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MVVtTlccJ8 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.hhTxzWkrsx + rm /tmp/tmp.MVVtTlccJ8 /tmp/tmp.hhTxzWkrsx + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.o1yvYpPn1V +++ mktemp ++ local LAST_ERR=/tmp/tmp.BhNKBKIE19 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.o1yvYpPn1V ++ cat /tmp/tmp.BhNKBKIE19 ++ rm /tmp/tmp.o1yvYpPn1V /tmp/tmp.BhNKBKIE19 ++ return 0 + wait_pod percona-server-mongodb-operator-559546695f-jh88x + local pod=percona-server-mongodb-operator-559546695f-jh88x + set +o xtrace waiting for pod/percona-server-mongodb-operator-559546695f-jh88x to be ready.OK + create_namespace balancer-19260 + local namespace=balancer-19260 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces balancer-19260' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces balancer-19260 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace balancer-19260 --ignore-not-found + xargs kubectl delete ns + awk '{print$1}' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.fGxTn5C9ZD ++ mktemp + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + local LAST_ERR=/tmp/tmp.6kX0aKC42h + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.XCjG6NOXGY ++ mktemp + local LAST_ERR=/tmp/tmp.RKLkVX9jVS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace balancer-19260 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fGxTn5C9ZD + cat /tmp/tmp.6kX0aKC42h + rm /tmp/tmp.fGxTn5C9ZD /tmp/tmp.6kX0aKC42h + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XCjG6NOXGY + cat /tmp/tmp.RKLkVX9jVS + rm /tmp/tmp.XCjG6NOXGY /tmp/tmp.RKLkVX9jVS + return 0 + kubectl_bin wait --for=delete namespace balancer-19260 ++ mktemp + local LAST_OUT=/tmp/tmp.95gWEUAsQU ++ mktemp + local LAST_ERR=/tmp/tmp.2s1R4ZDicx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace balancer-19260 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.95gWEUAsQU + cat /tmp/tmp.2s1R4ZDicx + rm /tmp/tmp.95gWEUAsQU /tmp/tmp.2s1R4ZDicx + return 0 + desc 'create namespace balancer-19260' + set +o xtrace ----------------------------------------------------------------------------------- create namespace balancer-19260 ----------------------------------------------------------------------------------- + kubectl_bin create namespace balancer-19260 ++ mktemp + local LAST_OUT=/tmp/tmp.PtS2FU6yZL ++ mktemp + local LAST_ERR=/tmp/tmp.A0Jcl5O2gb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace balancer-19260 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PtS2FU6yZL namespace/balancer-19260 created + cat /tmp/tmp.A0Jcl5O2gb + rm /tmp/tmp.PtS2FU6yZL /tmp/tmp.A0Jcl5O2gb + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.m8xpcf34rV +++ mktemp ++ local LAST_ERR=/tmp/tmp.abioIAUSr7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.m8xpcf34rV ++ cat /tmp/tmp.abioIAUSr7 ++ rm /tmp/tmp.m8xpcf34rV /tmp/tmp.abioIAUSr7 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2 --namespace=balancer-19260 ++ mktemp + local LAST_OUT=/tmp/tmp.zz1QfkMCR5 ++ mktemp + local LAST_ERR=/tmp/tmp.RnL9lWbwsP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2 --namespace=balancer-19260 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zz1QfkMCR5 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster2" modified. + cat /tmp/tmp.RnL9lWbwsP + rm /tmp/tmp.zz1QfkMCR5 /tmp/tmp.RnL9lWbwsP + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio Error: INSTALLATION FAILED: repo minio not found + [[ 1 -ge 10 ]] + (( n++ )) + sleep 60 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Mon Mar 24 03:29:52 2025 NAMESPACE: balancer-19260 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.balancer-19260.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace balancer-19260 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace balancer-19260 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace balancer-19260 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace balancer-19260 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rEfysqWpKL +++ mktemp ++ local LAST_ERR=/tmp/tmp.m1M2v7JmpP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rEfysqWpKL ++ cat /tmp/tmp.m1M2v7JmpP ++ rm /tmp/tmp.rEfysqWpKL /tmp/tmp.m1M2v7JmpP ++ return 0 + MINIO_POD=minio-service-847fc8bb8d-vcbxr + wait_pod minio-service-847fc8bb8d-vcbxr + local pod=minio-service-847fc8bb8d-vcbxr + set +o xtrace waiting for pod/minio-service-847fc8bb8d-vcbxr to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.balancer-19260.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.pUtjl5sYYV ++ mktemp + local LAST_ERR=/tmp/tmp.Abv4ZFiC0I + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.balancer-19260.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pUtjl5sYYV service/minio-service created + cat /tmp/tmp.Abv4ZFiC0I + rm /tmp/tmp.pUtjl5sYYV /tmp/tmp.Abv4ZFiC0I + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.mUXPrDobIe ++ mktemp + local LAST_ERR=/tmp/tmp.Yi2h3r2VC7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mUXPrDobIe make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.Yi2h3r2VC7 + rm /tmp/tmp.mUXPrDobIe /tmp/tmp.Yi2h3r2VC7 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.3DZ7c5k2LD ++ mktemp + local LAST_ERR=/tmp/tmp.h0CfWgN3JU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3DZ7c5k2LD secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.h0CfWgN3JU + rm /tmp/tmp.3DZ7c5k2LD /tmp/tmp.h0CfWgN3JU + return 0 + log 'create PSMDB cluster' + set +o xtrace [2025-03-24T03:30:31+0000] create PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/client-70.yml ++ mktemp + local LAST_OUT=/tmp/tmp.LLJiyPVR9R ++ mktemp + local LAST_ERR=/tmp/tmp.7Jd4pS3his + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/client-70.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LLJiyPVR9R secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.7Jd4pS3his + rm /tmp/tmp.LLJiyPVR9R /tmp/tmp.7Jd4pS3his + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/some-name-rs0.yml + kubectl_bin apply -f - + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/some-name-rs0.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AEEeZNeEG5 ++ mktemp + local LAST_ERR=/tmp/tmp.tOPRqHnbOV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AEEeZNeEG5 perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.tOPRqHnbOV + rm /tmp/tmp.AEEeZNeEG5 /tmp/tmp.tOPRqHnbOV + return 0 + log 'check if cfg pods started' + set +o xtrace [2025-03-24T03:30:36+0000] check if cfg pods started + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready........OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.....OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FykZM7T6mW +++ mktemp ++ local LAST_ERR=/tmp/tmp.V4jKaZYjDB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FykZM7T6mW ++ cat /tmp/tmp.V4jKaZYjDB ++ rm /tmp/tmp.FykZM7T6mW /tmp/tmp.V4jKaZYjDB ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ty0Il0TxnC +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ay69l1CMdE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ty0Il0TxnC ++ cat /tmp/tmp.Ay69l1CMdE ++ rm /tmp/tmp.ty0Il0TxnC /tmp/tmp.Ay69l1CMdE ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + log 'check if all shards started' + set +o xtrace [2025-03-24T03:31:56+0000] check if all shards started + wait_for_running some-name-rs0 3 false + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nPQ3Wdfl9c +++ mktemp ++ local LAST_ERR=/tmp/tmp.FBmvvv4LRI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nPQ3Wdfl9c ++ cat /tmp/tmp.FBmvvv4LRI ++ rm /tmp/tmp.nPQ3Wdfl9c /tmp/tmp.FBmvvv4LRI ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0IgA46Z8mf +++ mktemp ++ local LAST_ERR=/tmp/tmp.FmdqZ3TwHk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0IgA46Z8mf ++ cat /tmp/tmp.FmdqZ3TwHk ++ rm /tmp/tmp.0IgA46Z8mf /tmp/tmp.FmdqZ3TwHk ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-rs1 3 false + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gtJ9SSZcSJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.zQqXsNwzzb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gtJ9SSZcSJ ++ cat /tmp/tmp.zQqXsNwzzb ++ rm /tmp/tmp.gtJ9SSZcSJ /tmp/tmp.zQqXsNwzzb ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nAdsVH6AWJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.yMp55h6UOr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nAdsVH6AWJ ++ cat /tmp/tmp.yMp55h6UOr ++ rm /tmp/tmp.nAdsVH6AWJ /tmp/tmp.yMp55h6UOr ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-rs2 3 false + local name=some-name-rs2 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs2 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs2-0 + local pod=some-name-rs2-0 + set +o xtrace waiting for pod/some-name-rs2-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs2-1 + local pod=some-name-rs2-1 + set +o xtrace waiting for pod/some-name-rs2-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M9hWwm4hYi +++ mktemp ++ local LAST_ERR=/tmp/tmp.zUd4OSkOIQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M9hWwm4hYi ++ cat /tmp/tmp.zUd4OSkOIQ ++ rm /tmp/tmp.M9hWwm4hYi /tmp/tmp.zUd4OSkOIQ ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs2-2 + local pod=some-name-rs2-2 + set +o xtrace waiting for pod/some-name-rs2-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zelSQf32S7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hDgwx9bMrT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zelSQf32S7 ++ cat /tmp/tmp.hDgwx9bMrT ++ rm /tmp/tmp.zelSQf32S7 /tmp/tmp.hDgwx9bMrT ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + log 'check if mongos pods started' + set +o xtrace [2025-03-24T03:32:46+0000] check if mongos pods started + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mllsm587IL +++ mktemp ++ local LAST_ERR=/tmp/tmp.8au6qvODKb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mllsm587IL ++ cat /tmp/tmp.8au6qvODKb ++ rm /tmp/tmp.mllsm587IL /tmp/tmp.8au6qvODKb ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready...OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xU5o5a6J6J +++ mktemp ++ local LAST_ERR=/tmp/tmp.k9GB8d7CYf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xU5o5a6J6J ++ cat /tmp/tmp.k9GB8d7CYf ++ rm /tmp/tmp.xU5o5a6J6J /tmp/tmp.k9GB8d7CYf ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + check_balancer some-name true 10 + local cluster=some-name + local expected=true + local delay=10 + local balancer_running + log 'sleeping for 10 seconds...' + set +o xtrace [2025-03-24T03:33:36+0000] sleeping for 10 seconds... + sleep 10 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E -v 'Warning|cfg' ++ grep -E 'true|false' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.omvOp3vO3Y ++++ mktemp +++ local LAST_ERR=/tmp/tmp.m8XfdqFBy2 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.omvOp3vO3Y +++ cat /tmp/tmp.m8XfdqFBy2 +++ rm /tmp/tmp.omvOp3vO3Y /tmp/tmp.m8XfdqFBy2 +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NEjBB22sQw +++ mktemp ++ local LAST_ERR=/tmp/tmp.KpymacgHgj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NEjBB22sQw ++ cat /tmp/tmp.KpymacgHgj ++ rm /tmp/tmp.NEjBB22sQw /tmp/tmp.KpymacgHgj ++ return 0 + balancer_running=true + echo -n 'checking if balancer status is true...' checking if balancer status is true...+ [[ true != \t\r\u\e ]] + echo OK OK + write_data some-name + local cluster=some-name + log 'create user' + set +o xtrace [2025-03-24T03:33:51+0000] create user + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.balancer-19260 mongodb .svc.cluster.local --quiet 27017 mongosh + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.balancer-19260 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag=--quiet + local port=27017 + local mongo_bin=mongosh ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hunGCHj5CP +++ mktemp ++ local LAST_ERR=/tmp/tmp.6wvO6ilKDY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hunGCHj5CP ++ cat /tmp/tmp.6wvO6ilKDY ++ rm /tmp/tmp.hunGCHj5CP /tmp/tmp.6wvO6ilKDY ++ return 0 + local client_container=psmdb-client-58699fd5b7-xg89k + kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongosh mongodb://userAdmin:userAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.puJkLkbQxK ++ mktemp + local LAST_ERR=/tmp/tmp.gceZQ1MoCr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongosh mongodb://userAdmin:userAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.puJkLkbQxK [direct: mongos] admin> { ok: 1, '$clusterTime': { clusterTime: Timestamp({ t: 1742787234, i: 4 }), signature: { hash: Binary.createFromBase64('h3vBZA5vqNBl1xTtOteOuQETi10=', 0), keyId: Long('7485213903333359639') } }, operationTime: Timestamp({ t: 1742787234, i: 4 }) } [direct: mongos] admin> + cat /tmp/tmp.gceZQ1MoCr + rm /tmp/tmp.puJkLkbQxK /tmp/tmp.gceZQ1MoCr + return 0 + sleep 2 + log 'enable sharding' + set +o xtrace [2025-03-24T03:33:56+0000] enable sharding + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260 mongodb .svc.cluster.local --quiet 27017 mongosh + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag=--quiet + local port=27017 + local mongo_bin=mongosh ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.x4cKsTJ5QL +++ mktemp ++ local LAST_ERR=/tmp/tmp.PIskkGNUKb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.x4cKsTJ5QL ++ cat /tmp/tmp.PIskkGNUKb ++ rm /tmp/tmp.x4cKsTJ5QL /tmp/tmp.PIskkGNUKb ++ return 0 + local client_container=psmdb-client-58699fd5b7-xg89k + kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.oowdgF51Dt ++ mktemp + local LAST_ERR=/tmp/tmp.QsJRp1hNLy + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oowdgF51Dt [direct: mongos] admin> { ok: 1, '$clusterTime': { clusterTime: Timestamp({ t: 1742787241, i: 8 }), signature: { hash: Binary.createFromBase64('ipoK67T22Jeh4+d3gXIcO6ec8EI=', 0), keyId: Long('7485213903333359639') } }, operationTime: Timestamp({ t: 1742787241, i: 5 }) } [direct: mongos] admin> + cat /tmp/tmp.QsJRp1hNLy + rm /tmp/tmp.oowdgF51Dt /tmp/tmp.QsJRp1hNLy + return 0 + sleep 2 + log 'shard collection' + set +o xtrace [2025-03-24T03:34:03+0000] shard collection + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260 mongodb .svc.cluster.local --quiet 27017 mongosh + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag=--quiet + local port=27017 + local mongo_bin=mongosh ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jznF4cBKJJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.grd4pp99eS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jznF4cBKJJ ++ cat /tmp/tmp.grd4pp99eS ++ rm /tmp/tmp.jznF4cBKJJ /tmp/tmp.grd4pp99eS ++ return 0 + local client_container=psmdb-client-58699fd5b7-xg89k + kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' ++ mktemp + local LAST_OUT=/tmp/tmp.UW6PX9L8la ++ mktemp + local LAST_ERR=/tmp/tmp.RO54mTy8ra + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongosh mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.balancer-19260.svc.cluster.local:27017/admin --quiet' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UW6PX9L8la [direct: mongos] admin> { collectionsharded: 'app.city', ok: 1, '$clusterTime': { clusterTime: Timestamp({ t: 1742787247, i: 26 }), signature: { hash: Binary.createFromBase64('dyj6vVy1zPUuiSkeL6Ejt94q4u4=', 0), keyId: Long('7485213903333359639') } }, operationTime: Timestamp({ t: 1742787247, i: 25 }) } [direct: mongos] admin> + cat /tmp/tmp.RO54mTy8ra + rm /tmp/tmp.UW6PX9L8la /tmp/tmp.RO54mTy8ra + return 0 + log 'write data (this can take some time, be patient)' + set +o xtrace [2025-03-24T03:34:07+0000] write data (this can take some time, be patient) + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/data.js user:pass@some-name-mongos.balancer-19260 mongodb .svc.cluster.local --quiet mongosh + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/data.js + local uri=user:pass@some-name-mongos.balancer-19260 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag=--quiet + local mongo_bin=mongosh ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pAx3pewp9e +++ mktemp ++ local LAST_ERR=/tmp/tmp.oSn8Uhaukf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pAx3pewp9e ++ cat /tmp/tmp.oSn8Uhaukf ++ rm /tmp/tmp.pAx3pewp9e /tmp/tmp.oSn8Uhaukf ++ return 0 + local client_container=psmdb-client-58699fd5b7-xg89k ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/data.js balancer-19260/psmdb-client-58699fd5b7-xg89k:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.ca1N7Khgfi ++ mktemp + local LAST_ERR=/tmp/tmp.b3kEwc3Lzq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/data.js balancer-19260/psmdb-client-58699fd5b7-xg89k:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ca1N7Khgfi + cat /tmp/tmp.b3kEwc3Lzq + rm /tmp/tmp.ca1N7Khgfi /tmp/tmp.b3kEwc3Lzq + return 0 + kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'mongosh mongodb://user:pass@some-name-mongos.balancer-19260.svc.cluster.local/admin --quiet /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.Qbv4eHHgHB ++ mktemp + local LAST_ERR=/tmp/tmp.7bVLg2E2pl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'mongosh mongodb://user:pass@some-name-mongos.balancer-19260.svc.cluster.local/admin --quiet /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Qbv4eHHgHB + cat /tmp/tmp.7bVLg2E2pl + rm /tmp/tmp.Qbv4eHHgHB /tmp/tmp.7bVLg2E2pl + return 0 + check_backup_and_restore some-name 0 true + local cluster=some-name + local backup_suffix=0 + local balancer_end_state=true + local backup_name=backup-minio-0 + log 'running backup: backup-minio-0' + set +o xtrace [2025-03-24T04:00:19+0000] running backup: backup-minio-0 + run_backup minio backup-minio-0 + local storage=minio + local backup_name=backup-minio-0 + local type=logical + desc 'run backup backup-minio-0' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-0 ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-minio-0" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.9G7VyYefK5 ++ mktemp + local LAST_ERR=/tmp/tmp.LoLcwXItBH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9G7VyYefK5 perconaservermongodbbackup.psmdb.percona.com/backup-minio-0 created + cat /tmp/tmp.LoLcwXItBH + rm /tmp/tmp.9G7VyYefK5 /tmp/tmp.LoLcwXItBH + return 0 + wait_backup backup-minio-0 requested + local backup_name=backup-minio-0 + local target_state=requested + set +o xtrace waiting for backup-minio-0 to reach requested state. + log 'checking if balancer is disabled' + set +o xtrace [2025-03-24T04:00:25+0000] checking if balancer is disabled + check_balancer some-name false + local cluster=some-name + local expected=false + local delay=0 + local balancer_running + log 'sleeping for 0 seconds...' + set +o xtrace [2025-03-24T04:00:25+0000] sleeping for 0 seconds... + sleep 0 ++ grep -E 'true|false' ++ grep -E -v 'Warning|cfg' ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0rT2BYYQDS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.foGMZLYCYN +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.0rT2BYYQDS +++ cat /tmp/tmp.foGMZLYCYN +++ rm /tmp/tmp.0rT2BYYQDS /tmp/tmp.foGMZLYCYN +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AUAEpvYiT1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2pGfjgV9Fa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AUAEpvYiT1 ++ cat /tmp/tmp.2pGfjgV9Fa ++ rm /tmp/tmp.AUAEpvYiT1 /tmp/tmp.2pGfjgV9Fa ++ return 0 + balancer_running=false + echo -n 'checking if balancer status is false...' checking if balancer status is false...+ [[ false != \f\a\l\s\e ]] + echo OK OK + wait_backup backup-minio-0 ready + local backup_name=backup-minio-0 + local target_state=ready + set +o xtrace waiting for backup-minio-0 to reach ready state................. + log 'checking if balancer is true after backup' + set +o xtrace [2025-03-24T04:01:06+0000] checking if balancer is true after backup + check_balancer some-name true 10 + local cluster=some-name + local expected=true + local delay=10 + local balancer_running + log 'sleeping for 10 seconds...' + set +o xtrace [2025-03-24T04:01:06+0000] sleeping for 10 seconds... + sleep 10 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ grep -E 'true|false' ++++ mktemp ++ grep -E -v 'Warning|cfg' +++ local LAST_OUT=/tmp/tmp.koeExrWQg3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.X3W4qx1RbC +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.koeExrWQg3 +++ cat /tmp/tmp.X3W4qx1RbC +++ rm /tmp/tmp.koeExrWQg3 /tmp/tmp.X3W4qx1RbC +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gDT38Fvq9q +++ mktemp ++ local LAST_ERR=/tmp/tmp.9LFfOQyUjz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gDT38Fvq9q ++ cat /tmp/tmp.9LFfOQyUjz ++ rm /tmp/tmp.gDT38Fvq9q /tmp/tmp.9LFfOQyUjz ++ return 0 + balancer_running=true + echo -n 'checking if balancer status is true...' checking if balancer status is true...+ [[ true != \t\r\u\e ]] + echo OK OK + log 'running restore: restore-backup-minio-0' + set +o xtrace [2025-03-24T04:01:20+0000] running restore: restore-backup-minio-0 + run_restore backup-minio-0 + local backup_name=backup-minio-0 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/restore.yml + /usr/bin/sed -e 's/backupName:/backupName: backup-minio-0/' + /usr/bin/sed -e 's/name:/name: restore-backup-minio-0/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AbIoGHdezC ++ mktemp + local LAST_ERR=/tmp/tmp.leYe4dSKvB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AbIoGHdezC perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-0 created + cat /tmp/tmp.leYe4dSKvB + rm /tmp/tmp.AbIoGHdezC /tmp/tmp.leYe4dSKvB + return 0 + wait_restore backup-minio-0 some-name requested 0 + local backup_name=backup-minio-0 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace waiting psmdb-restore/restore-backup-minio-0 to reach requested state...............OK + '[' 0 -eq 1 ']' + log 'checking if balancer is disabled' + set +o xtrace [2025-03-24T04:01:45+0000] checking if balancer is disabled + check_balancer some-name false + local cluster=some-name + local expected=false + local delay=0 + local balancer_running + log 'sleeping for 0 seconds...' + set +o xtrace [2025-03-24T04:01:45+0000] sleeping for 0 seconds... + sleep 0 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ grep -E -v 'Warning|cfg' ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E 'true|false' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RJal1Q5OCu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6jJrqyE4El +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.RJal1Q5OCu +++ cat /tmp/tmp.6jJrqyE4El +++ rm /tmp/tmp.RJal1Q5OCu /tmp/tmp.6jJrqyE4El +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aqXKiqD8YM +++ mktemp ++ local LAST_ERR=/tmp/tmp.B6RsGds7kN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aqXKiqD8YM ++ cat /tmp/tmp.B6RsGds7kN ++ rm /tmp/tmp.aqXKiqD8YM /tmp/tmp.B6RsGds7kN ++ return 0 + balancer_running=false + echo -n 'checking if balancer status is false...' checking if balancer status is false...+ [[ false != \f\a\l\s\e ]] + echo OK OK + wait_restore backup-minio-0 some-name ready 1 + local backup_name=backup-minio-0 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace waiting psmdb-restore/restore-backup-minio-0 to reach ready state........................................................................OK + '[' 1 -eq 1 ']' + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aBXaffDJl4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ccqQKPddPW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aBXaffDJl4 ++ cat /tmp/tmp.ccqQKPddPW ++ rm /tmp/tmp.aBXaffDJl4 /tmp/tmp.ccqQKPddPW ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TKVeYFBgmi +++ mktemp ++ local LAST_ERR=/tmp/tmp.GEO8E1OUot ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TKVeYFBgmi ++ cat /tmp/tmp.GEO8E1OUot ++ rm /tmp/tmp.TKVeYFBgmi /tmp/tmp.GEO8E1OUot ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4FRm1TxQgT +++ mktemp ++ local LAST_ERR=/tmp/tmp.e96CG8d0rR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4FRm1TxQgT ++ cat /tmp/tmp.e96CG8d0rR ++ rm /tmp/tmp.4FRm1TxQgT /tmp/tmp.e96CG8d0rR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y70GQlxibX +++ mktemp ++ local LAST_ERR=/tmp/tmp.wD8HUelRep ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.y70GQlxibX ++ cat /tmp/tmp.wD8HUelRep ++ rm /tmp/tmp.y70GQlxibX /tmp/tmp.wD8HUelRep ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fS04MqbAOF +++ mktemp ++ local LAST_ERR=/tmp/tmp.wwfZoIvS53 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fS04MqbAOF ++ cat /tmp/tmp.wwfZoIvS53 ++ rm /tmp/tmp.fS04MqbAOF /tmp/tmp.wwfZoIvS53 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + log 'checking if balancer is true after restore' + set +o xtrace [2025-03-24T04:04:35+0000] checking if balancer is true after restore + check_balancer some-name true 10 + local cluster=some-name + local expected=true + local delay=10 + local balancer_running + log 'sleeping for 10 seconds...' + set +o xtrace [2025-03-24T04:04:35+0000] sleeping for 10 seconds... + sleep 10 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E -v 'Warning|cfg' ++ grep -E 'true|false' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RA8VaCTaII ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FD2HhrBpBQ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.RA8VaCTaII +++ cat /tmp/tmp.FD2HhrBpBQ +++ rm /tmp/tmp.RA8VaCTaII /tmp/tmp.FD2HhrBpBQ +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tAdYWFYHaG +++ mktemp ++ local LAST_ERR=/tmp/tmp.mFocDrMofk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tAdYWFYHaG ++ cat /tmp/tmp.mFocDrMofk ++ rm /tmp/tmp.tAdYWFYHaG /tmp/tmp.mFocDrMofk ++ return 0 + balancer_running=true + echo -n 'checking if balancer status is true...' checking if balancer status is true...+ [[ true != \t\r\u\e ]] + echo OK OK + log 'disabling balancer' + set +o xtrace [2025-03-24T04:04:49+0000] disabling balancer + kubectl patch psmdb some-name --type=merge -p '{"spec":{"sharding":{"balancer":{"enabled":false}}}}' perconaservermongodb.psmdb.percona.com/some-name patched + check_balancer some-name false 10 + local cluster=some-name + local expected=false + local delay=10 + local balancer_running + log 'sleeping for 10 seconds...' + set +o xtrace [2025-03-24T04:04:51+0000] sleeping for 10 seconds... + sleep 10 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E -v 'Warning|cfg' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp ++ grep -E 'true|false' +++ local LAST_OUT=/tmp/tmp.qU1O2EugOC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qOD3aKIfeO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qU1O2EugOC +++ cat /tmp/tmp.qOD3aKIfeO +++ rm /tmp/tmp.qU1O2EugOC /tmp/tmp.qOD3aKIfeO +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s4X1rpf4z8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ChDYGHiPZi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s4X1rpf4z8 ++ cat /tmp/tmp.ChDYGHiPZi ++ rm /tmp/tmp.s4X1rpf4z8 /tmp/tmp.ChDYGHiPZi ++ return 0 + balancer_running=false + echo -n 'checking if balancer status is false...' checking if balancer status is false...+ [[ false != \f\a\l\s\e ]] + echo OK OK + check_backup_and_restore some-name 1 false + local cluster=some-name + local backup_suffix=1 + local balancer_end_state=false + local backup_name=backup-minio-1 + log 'running backup: backup-minio-1' + set +o xtrace [2025-03-24T04:05:05+0000] running backup: backup-minio-1 + run_backup minio backup-minio-1 + local storage=minio + local backup_name=backup-minio-1 + local type=logical + desc 'run backup backup-minio-1' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-1 ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-minio-1" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.54VkWdAmIi ++ mktemp + local LAST_ERR=/tmp/tmp.8LhnrPz7rq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.54VkWdAmIi perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.8LhnrPz7rq + rm /tmp/tmp.54VkWdAmIi /tmp/tmp.8LhnrPz7rq + return 0 + wait_backup backup-minio-1 requested + local backup_name=backup-minio-1 + local target_state=requested + set +o xtrace waiting for backup-minio-1 to reach requested state. + log 'checking if balancer is disabled' + set +o xtrace [2025-03-24T04:05:09+0000] checking if balancer is disabled + check_balancer some-name false + local cluster=some-name + local expected=false + local delay=0 + local balancer_running + log 'sleeping for 0 seconds...' + set +o xtrace [2025-03-24T04:05:09+0000] sleeping for 0 seconds... + sleep 0 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E 'true|false' ++ grep -E -v 'Warning|cfg' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aInJN3f5o8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.f0xJp4qq1n +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.aInJN3f5o8 +++ cat /tmp/tmp.f0xJp4qq1n +++ rm /tmp/tmp.aInJN3f5o8 /tmp/tmp.f0xJp4qq1n +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fMGOVEleAX +++ mktemp ++ local LAST_ERR=/tmp/tmp.RDJ6nz8tT1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fMGOVEleAX ++ cat /tmp/tmp.RDJ6nz8tT1 ++ rm /tmp/tmp.fMGOVEleAX /tmp/tmp.RDJ6nz8tT1 ++ return 0 + balancer_running=false + echo -n 'checking if balancer status is false...' checking if balancer status is false...+ [[ false != \f\a\l\s\e ]] + echo OK OK + wait_backup backup-minio-1 ready + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state............. + log 'checking if balancer is false after backup' + set +o xtrace [2025-03-24T04:05:40+0000] checking if balancer is false after backup + check_balancer some-name false 10 + local cluster=some-name + local expected=false + local delay=10 + local balancer_running + log 'sleeping for 10 seconds...' + set +o xtrace [2025-03-24T04:05:40+0000] sleeping for 10 seconds... + sleep 10 ++ run_mongosh 'sh.getBalancerState()' clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ grep -E -v 'Warning|cfg' ++ local 'command=sh.getBalancerState()' ++ local uri=clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ grep -E 'true|false' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kECH08nQSV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Le4c69Fkjv +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kECH08nQSV +++ cat /tmp/tmp.Le4c69Fkjv +++ rm /tmp/tmp.kECH08nQSV /tmp/tmp.Le4c69Fkjv +++ return 0 ++ local client_container=psmdb-client-58699fd5b7-xg89k ++ [[ clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260 == *cfg* ]] ++ replica_set=cfg ++ kubectl_bin exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bAVFCaGVtA +++ mktemp ++ local LAST_ERR=/tmp/tmp.r5vp4aR9Ws ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-58699fd5b7-xg89k -- bash -c 'printf '\''sh.getBalancerState()\n'\'' | mongosh --quiet mongodb+srv://clusterAdmin:clusterAdmin123456@some-name-cfg.balancer-19260.svc.cluster.local/admin?ssl=false\&replicaSet=cfg ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bAVFCaGVtA ++ cat /tmp/tmp.r5vp4aR9Ws ++ rm /tmp/tmp.bAVFCaGVtA /tmp/tmp.r5vp4aR9Ws ++ return 0 + balancer_running=false + echo -n 'checking if balancer status is false...' checking if balancer status is false...+ [[ false != \f\a\l\s\e ]] + echo OK OK + log 'running restore: restore-backup-minio-1' + set +o xtrace [2025-03-24T04:05:53+0000] running restore: restore-backup-minio-1 + run_restore backup-minio-1 + local backup_name=backup-minio-1 + /usr/bin/sed -e 's/name:/name: restore-backup-minio-1/' + /usr/bin/sed -e 's/backupName:/backupName: backup-minio-1/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/balancer/conf/restore.yml ++ mktemp + local LAST_OUT=/tmp/tmp.OY8lmASnV9 ++ mktemp + local LAST_ERR=/tmp/tmp.XNnScW8hd2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OY8lmASnV9 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created + cat /tmp/tmp.XNnScW8hd2 + rm /tmp/tmp.OY8lmASnV9 /tmp/tmp.XNnScW8hd2 + return 0 + wait_restore backup-minio-1 some-name requested 0 + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace waiting psmdb-restore/restore-backup-minio-1 to reach requested state....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2025-03-24T04:05:22.630Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "a94da1fc-3b36-4a7b-8b3e-d4f0561b4056", "backup": "backup-minio-1", "pbmName": "2025-03-24T04:05:07Z", "meta": {"type":"logical","opid":"67e0d9f3b93b1393395a4353","name":"2025-03-24T04:05:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"cfg","backup_name":"2025-03-24T04:05:07Z/cfg/metadata.json","oplog_name":"2025-03-24T04:05:07Z/cfg/oplog","start_ts":1742789108,"status":"dumpDone","iscs":true,"last_transition_ts":1742789110,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":0,"I":0},"node":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs0","backup_name":"2025-03-24T04:05:07Z/rs0/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs0/oplog","start_ts":1742789108,"status":"dumpDone","last_transition_ts":1742789110,"first_write_ts":{"T":1742789106,"I":1},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs1","backup_name":"2025-03-24T04:05:07Z/rs1/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs1/oplog","start_ts":1742789108,"status":"running","last_transition_ts":1742789108,"first_write_ts":{"T":1742789102,"I":2},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs2","backup_name":"2025-03-24T04:05:07Z/rs2/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs2/oplog","start_ts":1742789108,"status":"dumpDone","last_transition_ts":1742789111,"first_write_ts":{"T":1742789106,"I":6},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789111,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.4-2","fcv":"8.0","start_ts":1742789108,"last_transition_ts":1742789110,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1,"I":1},"hb":{"T":1742789118,"I":2},"status":"running","conditions":[{"timestamp":1742789108,"status":"starting"},{"timestamp":1742789110,"status":"running"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.balancer-19260.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.balancer-19260.svc.cluster.local:27017","some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-19260.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"}],"pbm_version":"2.9.0","balancer":"off"}} 2025-03-24T04:05:22.632Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "a94da1fc-3b36-4a7b-8b3e-d4f0561b4056"} 2025-03-24T04:05:27.633Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fe61de18-bf6c-43c6-b2bc-ccaf8d428e0a"} 2025-03-24T04:05:27.755Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fe61de18-bf6c-43c6-b2bc-ccaf8d428e0a", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:05:27.756Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fe61de18-bf6c-43c6-b2bc-ccaf8d428e0a", "currentJob": {"Name":"backup-minio-1","Type":0}} 2025-03-24T04:05:27.893Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fe61de18-bf6c-43c6-b2bc-ccaf8d428e0a", "backup": "backup-minio-1", "pbmName": "2025-03-24T04:05:07Z", "meta": {"type":"logical","opid":"67e0d9f3b93b1393395a4353","name":"2025-03-24T04:05:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"cfg","backup_name":"2025-03-24T04:05:07Z/cfg/metadata.json","oplog_name":"2025-03-24T04:05:07Z/cfg/oplog","start_ts":1742789108,"status":"dumpDone","iscs":true,"last_transition_ts":1742789110,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1742789123,"I":13},"node":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs0","backup_name":"2025-03-24T04:05:07Z/rs0/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs0/oplog","start_ts":1742789108,"status":"dumpDone","last_transition_ts":1742789110,"first_write_ts":{"T":1742789106,"I":1},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs1","backup_name":"2025-03-24T04:05:07Z/rs1/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs1/oplog","start_ts":1742789108,"status":"dumpDone","last_transition_ts":1742789123,"first_write_ts":{"T":1742789102,"I":2},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs2","backup_name":"2025-03-24T04:05:07Z/rs2/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs2/oplog","start_ts":1742789108,"status":"dumpDone","last_transition_ts":1742789111,"first_write_ts":{"T":1742789106,"I":6},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789111,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.4-2","fcv":"8.0","start_ts":1742789108,"last_transition_ts":1742789123,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1,"I":1},"hb":{"T":1742789123,"I":7},"status":"dumpDone","conditions":[{"timestamp":1742789108,"status":"starting"},{"timestamp":1742789110,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.balancer-19260.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.balancer-19260.svc.cluster.local:27017","some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-19260.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"}],"pbm_version":"2.9.0","balancer":"off"}} 2025-03-24T04:05:27.898Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fe61de18-bf6c-43c6-b2bc-ccaf8d428e0a"} 2025-03-24T04:05:32.899Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "0dba6fb6-9ee0-4d39-9409-7a3a9aeb95bd"} 2025-03-24T04:05:32.972Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "0dba6fb6-9ee0-4d39-9409-7a3a9aeb95bd", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:05:32.972Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "0dba6fb6-9ee0-4d39-9409-7a3a9aeb95bd", "currentJob": {"Name":"backup-minio-1","Type":0}} 2025-03-24T04:05:33.050Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "0dba6fb6-9ee0-4d39-9409-7a3a9aeb95bd", "backup": "backup-minio-1", "pbmName": "2025-03-24T04:05:07Z", "meta": {"type":"logical","opid":"67e0d9f3b93b1393395a4353","name":"2025-03-24T04:05:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"cfg","backup_name":"2025-03-24T04:05:07Z/cfg/metadata.json","oplog_name":"2025-03-24T04:05:07Z/cfg/oplog","start_ts":1742789108,"status":"dumpDone","iscs":true,"last_transition_ts":1742789110,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1742789123,"I":13},"node":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs0","backup_name":"2025-03-24T04:05:07Z/rs0/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs0/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789129,"first_write_ts":{"T":1742789106,"I":1},"last_write_ts":{"T":1742789121,"I":1},"node":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"},{"timestamp":1742789129,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs1","backup_name":"2025-03-24T04:05:07Z/rs1/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs1/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789128,"first_write_ts":{"T":1742789102,"I":2},"last_write_ts":{"T":1742789113,"I":14},"node":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"},{"timestamp":1742789128,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs2","backup_name":"2025-03-24T04:05:07Z/rs2/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs2/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789129,"first_write_ts":{"T":1742789106,"I":6},"last_write_ts":{"T":1742789121,"I":8},"node":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789111,"status":"dumpDone"},{"timestamp":1742789129,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":26300237,"mongodb_version":"8.0.4-2","fcv":"8.0","start_ts":1742789108,"last_transition_ts":1742789123,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1,"I":1},"hb":{"T":1742789128,"I":6},"status":"dumpDone","conditions":[{"timestamp":1742789108,"status":"starting"},{"timestamp":1742789110,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.balancer-19260.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.balancer-19260.svc.cluster.local:27017","some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-19260.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"}],"pbm_version":"2.9.0","balancer":"off"}} 2025-03-24T04:05:33.053Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "0dba6fb6-9ee0-4d39-9409-7a3a9aeb95bd"} 2025-03-24T04:05:38.054Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc"} 2025-03-24T04:05:38.138Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:05:38.138Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc", "currentJob": {"Name":"backup-minio-1","Type":0}} 2025-03-24T04:05:38.225Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc", "backup": "backup-minio-1", "pbmName": "2025-03-24T04:05:07Z", "meta": {"type":"logical","opid":"67e0d9f3b93b1393395a4353","name":"2025-03-24T04:05:07Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"cfg","backup_name":"2025-03-24T04:05:07Z/cfg/metadata.json","oplog_name":"2025-03-24T04:05:07Z/cfg/oplog","start_ts":1742789108,"status":"done","iscs":true,"last_transition_ts":1742789134,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1742789123,"I":13},"node":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"},{"timestamp":1742789134,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs0","backup_name":"2025-03-24T04:05:07Z/rs0/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs0/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789129,"first_write_ts":{"T":1742789106,"I":1},"last_write_ts":{"T":1742789121,"I":1},"node":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789110,"status":"dumpDone"},{"timestamp":1742789129,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs1","backup_name":"2025-03-24T04:05:07Z/rs1/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs1/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789128,"first_write_ts":{"T":1742789102,"I":2},"last_write_ts":{"T":1742789113,"I":14},"node":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"},{"timestamp":1742789128,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"},{"name":"rs2","backup_name":"2025-03-24T04:05:07Z/rs2/metadata.json","oplog_name":"2025-03-24T04:05:07Z/rs2/oplog","start_ts":1742789108,"status":"done","last_transition_ts":1742789129,"first_write_ts":{"T":1742789106,"I":6},"last_write_ts":{"T":1742789121,"I":8},"node":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017","conditions":[{"timestamp":1742789108,"status":"running"},{"timestamp":1742789111,"status":"dumpDone"},{"timestamp":1742789129,"status":"done"}],"pbm_version":"2.9.0","mongo_version":"8.0.4-2"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":26446465,"mongodb_version":"8.0.4-2","fcv":"8.0","start_ts":1742789108,"last_transition_ts":1742789135,"first_write_ts":{"T":1742789108,"I":31},"last_write_ts":{"T":1742789113,"I":14},"hb":{"T":1742789133,"I":2},"status":"done","conditions":[{"timestamp":1742789108,"status":"starting"},{"timestamp":1742789110,"status":"running"},{"timestamp":1742789123,"status":"dumpDone"},{"timestamp":1742789135,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.balancer-19260.svc.cluster.local:27017"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.balancer-19260.svc.cluster.local:27017","some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs1-1.some-name-rs1.balancer-19260.svc.cluster.local:27017"},{"rs":"rs2","n":["some-name-rs2-1.some-name-rs2.balancer-19260.svc.cluster.local:27017","some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs2-2.some-name-rs2.balancer-19260.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-2.some-name-rs0.balancer-19260.svc.cluster.local:27017","some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.balancer-19260.svc.cluster.local:27017"}],"pbm_version":"2.9.0","balancer":"off"}} 2025-03-24T04:05:38.226Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc", "previous": "running", "current": "ready"} 2025-03-24T04:05:38.238Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "f5c25dfb-bb25-4365-acda-d6632c4c2dbc"} 2025-03-24T04:05:38.238Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "8c1ae295-9bdb-4518-baf2-3f10244dcd36"} 2025-03-24T04:05:38.238Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "8c1ae295-9bdb-4518-baf2-3f10244dcd36"} 2025-03-24T04:05:43.238Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fa3d288d-3527-490f-b2ef-e43624fb5ae3"} 2025-03-24T04:05:43.238Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "backup-minio-1", "reconcileID": "fa3d288d-3527-490f-b2ef-e43624fb5ae3"} 2025-03-24T04:05:55.999Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "2a36c820-9cd6-42a2-8733-9cb8c575dbcc", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:05:56.144Z INFO Terminating mongos pods {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "2a36c820-9cd6-42a2-8733-9cb8c575dbcc"} 2025-03-24T04:06:00.753Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "625d1cd9-924c-48c6-b197-74a8f498c6bc", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:00.753Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "625d1cd9-924c-48c6-b197-74a8f498c6bc", "previous": "ready", "current": "initializing"} 2025-03-24T04:06:01.163Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "6edf57c3-e884-476b-a6f1-4cb63c431bf6", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:01.285Z INFO Waiting for mongos pods to terminate {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "6edf57c3-e884-476b-a6f1-4cb63c431bf6"} 2025-03-24T04:06:03.115Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "e05acd4d-afa6-46fb-b6b1-6a9ed3754512", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:06.286Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "90b13eef-b9a4-4f73-b36a-e4fe10f0f58c", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:06.413Z INFO Waiting for mongos pods to terminate {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "90b13eef-b9a4-4f73-b36a-e4fe10f0f58c"} 2025-03-24T04:06:07.578Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "f64ac014-ab4a-4f55-acc5-c2cb6e178901", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:11.413Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "e62c1023-4cc2-483a-a5df-dcaa0753b33b", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:11.531Z INFO Waiting for mongos pods to terminate {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "e62c1023-4cc2-483a-a5df-dcaa0753b33b"} 2025-03-24T04:06:14.761Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "f0726ffe-2762-4449-a7fd-74497f51cda8", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:16.532Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "aedd36db-2cb7-4fc3-ae18-23bf7fa5f128", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:16.725Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "aedd36db-2cb7-4fc3-ae18-23bf7fa5f128", "backup": "2025-03-24T04:05:07Z"} 2025-03-24T04:06:16.725Z INFO Starting logical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "aedd36db-2cb7-4fc3-ae18-23bf7fa5f128", "backup": "2025-03-24T04:05:07Z"} 2025-03-24T04:06:16.725Z INFO Sending restore command {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "aedd36db-2cb7-4fc3-ae18-23bf7fa5f128", "restoreCmd": "name: 2025-03-24T04:06:16.725783831Z, snapshot: 2025-03-24T04:05:07Z"} 2025-03-24T04:06:16.729Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "aedd36db-2cb7-4fc3-ae18-23bf7fa5f128", "previous": "", "current": "requested"} 2025-03-24T04:06:17.296Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "c31c4a82-0de9-4b5e-8047-addf20c2e595", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:17.365Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "c31c4a82-0de9-4b5e-8047-addf20c2e595", "previous": "requested", "current": "running"} 2025-03-24T04:06:17.920Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "828894ad-c514-4d0f-8293-2e7cfc165b13", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:22.297Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "6b3eca9a-0a3b-4582-b5d4-ff57befd16ae", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:23.510Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "ca55ddbf-b0e9-4f86-8173-cd75d8d09629", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:27.419Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "b22a9383-410b-49b6-8d4f-30434d69e496", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:31.823Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "16218730-66ef-41c8-b6d3-0d5a982862ec", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:32.537Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "86e9048b-e069-448f-aad6-228c2a1507f6", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:37.643Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "c741de01-3689-4cf2-ab8c-db44fb47723c", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:41.122Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "40ccffcb-30da-48f7-b1bd-fd620e8cd00d", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:42.779Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "186f4b61-7066-46e7-9112-c036c9e56f34", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:47.923Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "65860b55-37ea-417d-82c7-47fbee31fca6", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:48.766Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "0937d782-9d65-461a-93c5-a8ad036cbb01", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:53.070Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "81455294-6828-4c7a-8c36-8ef986f9a01e", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:06:57.684Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "71033b61-e519-4f74-a7c5-78d63fc2dc8c", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:06:58.193Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "d7490c78-71ef-4a1d-a691-14dbb3f74655", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:03.304Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "573efdd7-c146-4f8c-aaca-c2ed32797dea", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:05.654Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "e0b4f0a4-653c-4d77-bb5e-9008e072920f", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:08.419Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "321f748a-2c39-46c0-8694-52bc502b762c", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:13.518Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "8734c202-4ecf-49b4-8c6f-e861601891ef", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:13.792Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "db71ca67-9781-4419-a5a9-197d7388e992", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:18.631Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "591630fa-2a05-4ed8-bedc-71fd072a63b2", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:22.490Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "6c84ea19-0a61-4342-affa-b764842cdd9e", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:23.736Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "e3385d71-af04-4192-8fa6-26257273c0d1", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:28.877Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "3e112178-8345-40c2-9654-ea3433c90aed", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:31.059Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "ff486ff1-a88e-4e34-926a-e9d905046743", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:34.204Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "feecc1f7-a62c-430b-95a7-8fa4c89ecffc", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:39.320Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "a1a6134e-c904-497b-8a6b-51edb993a08f", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:39.407Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "053cf04d-524b-4b7c-b495-d0d0c09b8313", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:44.436Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "7bd993f9-be4e-4b69-8c9c-804e9fb4a3ff", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:47.676Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "562791e3-73a2-4b0f-b92f-879424a7a7ff", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:49.536Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "e50ae0fe-7997-4e69-900f-70477405c6b7", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:54.665Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "bf807b51-e407-4b96-a9e3-b649bccdae41", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:07:55.973Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "9383542a-5c14-4bb7-be0d-2e6c79609eab", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:07:59.743Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "1bf1ce1e-96d1-4858-ad00-0d827fbddb75", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:08:03.614Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "67ba005c-90d2-4ba1-bc4a-fc08b01bc86e", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:04.834Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "3f9aff7b-1101-4fc6-a418-fe38543ab619", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:08:09.945Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "8926a121-ffe9-48c2-928d-48b70d5dea21", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:08:12.846Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "a9a773ef-b4d9-4f8f-9cbd-87ae6c65cd6d", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:15.077Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "01b9bc7a-6430-4836-abf6-dd0aae7ca0b6", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:08:19.669Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "b84c8790-597c-401e-b581-40212a9de0a9", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:20.166Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "7854bfb1-87cc-44b7-af7f-a81e5e472012", "cluster": "some-name", "namespace": "balancer-19260"} 2025-03-24T04:08:20.308Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-minio-1","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "restore-backup-minio-1", "reconcileID": "7854bfb1-87cc-44b7-af7f-a81e5e472012", "previous": "running", "current": "ready"} 2025-03-24T04:08:27.343Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "bd364b56-0ebe-44d4-890d-549372100209", "name": "some-name-mongos", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "generation": 0, "resourceVersion": ""} 2025-03-24T04:08:27.372Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "bd364b56-0ebe-44d4-890d-549372100209", "name": "some-name-mongos-", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "generation": 0, "resourceVersion": ""} 2025-03-24T04:08:37.502Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "bd364b56-0ebe-44d4-890d-549372100209", "error": "reconcile mongos: failed to disable balancer: failed to get mongos connection: ping mongo: server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.balancer-19260.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.229.66:27017: connect: connection refused }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: some-name-mongos.balancer-19260.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp 34.118.229.66:27017: connect: connection refused }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).disableBalancer\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/balancer.go:144\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).disableBalancerIfNeeded\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/balancer.go:124\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileMongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:328\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:288\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:249\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).disableBalancer\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/balancer.go:146\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).disableBalancerIfNeeded\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/balancer.go:124\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileMongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:1209\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:328\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:288\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:249\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to disable balancer\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileMongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:1210\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:328\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:288\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:249\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile mongos\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:425\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:118\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:328\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:288\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:249\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:341 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:288 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.3/pkg/internal/controller/controller.go:249 2025-03-24T04:08:39.768Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "53a5a1ad-543b-4ee3-8401-911198b7a1a8", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:39.768Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "53a5a1ad-543b-4ee3-8401-911198b7a1a8", "previous": "error", "current": "initializing"} 2025-03-24T04:08:43.306Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "aac18883-41fc-495f-9dfc-c90595e8ee66", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:47.061Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "d27e9ea7-b39a-44e0-9fa8-33468a307da5", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:54.217Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "0f41187b-0348-405d-9608-ed866fc71ecc", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:08:57.979Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "e231c330-f13b-4fb7-aef3-6ece3e72ffab", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:09:01.370Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "2e25f24a-df98-40cf-a261-0d0e7e73ca5f", "upgradeInProgress": false, "replsetsReady": 4, "clusterState": "ready"} 2025-03-24T04:09:08.426Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"balancer-19260"}, "namespace": "balancer-19260", "name": "some-name", "reconcileID": "2731d18d-1f95-41ff-9ddf-a94784bebceb", "previous": "initializing", "current": "ready"} apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-minio-1","namespace":"balancer-19260"},"spec":{"backupName":"backup-minio-1","clusterName":"some-name"}} creationTimestamp: "2025-03-24T04:05:55Z" generation: 1 name: restore-backup-minio-1 namespace: balancer-19260 resourceVersion: "88684" uid: d55859d3-4c17-46ed-87d5-45e715891a36 spec: backupName: backup-minio-1 clusterName: some-name status: completed: "2025-03-24T04:08:18Z" pbmName: "2025-03-24T04:06:16.725783831Z" state: ready Restore object restore-backup-minio-1 is in ready state. something went wrong with operator or kubernetes cluster