Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/logs/scheduled-backup.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ create_infra scheduled-backup-28431
+ local ns=scheduled-backup-28431
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.jZWdcjdovi
++ mktemp
+ local LAST_ERR=/tmp/tmp.CQVS6IzrKj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jZWdcjdovi
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.CQVS6IzrKj
+ rm /tmp/tmp.jZWdcjdovi /tmp/tmp.CQVS6IzrKj
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-28690 cron-some-name-20251112103700-mnppz --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "cron-some-name-20251112103700-mnppz" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.0krjZStx8e
++ mktemp
+ local LAST_ERR=/tmp/tmp.KouNe0CX17
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0krjZStx8e
+ cat /tmp/tmp.KouNe0CX17
+ rm /tmp/tmp.0krjZStx8e /tmp/tmp.KouNe0CX17
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.moxaQq2urp
++ mktemp
+ local LAST_ERR=/tmp/tmp.mHiZuUlRLv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.moxaQq2urp
+ cat /tmp/tmp.mHiZuUlRLv
+ rm /tmp/tmp.moxaQq2urp /tmp/tmp.mHiZuUlRLv
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Brf78PtqKh
++ mktemp
+ local LAST_ERR=/tmp/tmp.oFJ0Xtp7qf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Brf78PtqKh
+ cat /tmp/tmp.oFJ0Xtp7qf
+ rm /tmp/tmp.Brf78PtqKh /tmp/tmp.oFJ0Xtp7qf
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.oZAQlWMtx1
++ mktemp
+ local LAST_ERR=/tmp/tmp.cy2LxFVkrm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oZAQlWMtx1
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.cy2LxFVkrm
+ rm /tmp/tmp.oZAQlWMtx1 /tmp/tmp.cy2LxFVkrm
+ return 0
+ check_crd_for_deletion PR-2084-c01795af
+ local git_tag=PR-2084-c01795af
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2084-c01795af/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AlAlOpLs1s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0z7p3Gewm7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AlAlOpLs1s
++ cat /tmp/tmp.0z7p3Gewm7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AlAlOpLs1s
++ cat /tmp/tmp.0z7p3Gewm7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.AlAlOpLs1s
++ cat /tmp/tmp.0z7p3Gewm7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.AlAlOpLs1s
++ cat /tmp/tmp.0z7p3Gewm7
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.AlAlOpLs1s /tmp/tmp.0z7p3Gewm7
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.OtoUzi1XAe
++ mktemp
+ local LAST_ERR=/tmp/tmp.DirW8ZGIO6
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.sYS36y9dhk
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.oDipCu3u1P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OtoUzi1XAe
+ cat /tmp/tmp.DirW8ZGIO6
+ rm /tmp/tmp.OtoUzi1XAe /tmp/tmp.DirW8ZGIO6
+ return 0
namespace "scheduled-backup-28690" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sYS36y9dhk
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.oDipCu3u1P
+ rm /tmp/tmp.sYS36y9dhk /tmp/tmp.oDipCu3u1P
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.autYY6vENs
++ mktemp
+ local LAST_ERR=/tmp/tmp.OmrAjGAMwR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.autYY6vENs
+ cat /tmp/tmp.OmrAjGAMwR
+ rm /tmp/tmp.autYY6vENs /tmp/tmp.OmrAjGAMwR
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.rD0iaEX2As
++ mktemp
+ local LAST_ERR=/tmp/tmp.0VxJznKLG1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rD0iaEX2As
namespace/psmdb-operator created
+ cat /tmp/tmp.0VxJznKLG1
+ rm /tmp/tmp.rD0iaEX2As /tmp/tmp.0VxJznKLG1
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oRMveTnYJO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7ZoRniB6kk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oRMveTnYJO
++ cat /tmp/tmp.7ZoRniB6kk
++ rm /tmp/tmp.oRMveTnYJO /tmp/tmp.7ZoRniB6kk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.z5RxpXiTBM
++ mktemp
+ local LAST_ERR=/tmp/tmp.JzUei8z9Jc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z5RxpXiTBM
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1" modified.
+ cat /tmp/tmp.JzUei8z9Jc
+ rm /tmp/tmp.z5RxpXiTBM /tmp/tmp.JzUei8z9Jc
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2084-c01795af'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2084-c01795af
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.63bCrMHL35
++ mktemp
+ local LAST_ERR=/tmp/tmp.zDP1HktmXo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.63bCrMHL35
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.zDP1HktmXo
+ rm /tmp/tmp.63bCrMHL35 /tmp/tmp.zDP1HktmXo
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.UqxhUO658T
++ mktemp
+ local LAST_ERR=/tmp/tmp.LFOgclplbh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UqxhUO658T
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.LFOgclplbh
+ rm /tmp/tmp.UqxhUO658T /tmp/tmp.LFOgclplbh
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2084-c01795af") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.WM9MLFdmd8
++ mktemp
+ local LAST_ERR=/tmp/tmp.rDv7lFlM7u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WM9MLFdmd8
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.rDv7lFlM7u
+ rm /tmp/tmp.WM9MLFdmd8 /tmp/tmp.rDv7lFlM7u
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4365JlHrs6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tJnUZ7LfOC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4365JlHrs6
++ cat /tmp/tmp.tJnUZ7LfOC
++ rm /tmp/tmp.4365JlHrs6 /tmp/tmp.tJnUZ7LfOC
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-68b58d889-m4lqv
+ local pod=percona-server-mongodb-operator-68b58d889-m4lqv
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-68b58d889-m4lqv to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KsYRPnkD7z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7q8n3sgUBP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KsYRPnkD7z
++ cat /tmp/tmp.7q8n3sgUBP
++ rm /tmp/tmp.KsYRPnkD7z /tmp/tmp.7q8n3sgUBP
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-68b58d889-m4lqv
++ mktemp
+ local LAST_OUT=/tmp/tmp.iqLBPMGHpT
++ mktemp
+ local LAST_ERR=/tmp/tmp.d1Qe34c6kD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-68b58d889-m4lqv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iqLBPMGHpT
+ cat /tmp/tmp.d1Qe34c6kD
+ rm /tmp/tmp.iqLBPMGHpT /tmp/tmp.d1Qe34c6kD
+ return 0
2025-11-12T10:41:05.646Z	INFO	setup	Manager starting up	{"gitCommit": "c01795afa449ea1179a20e20bf96b3936f20ad1e", "gitBranch": "PR-2084-c01795af", "buildTime": "", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
+ create_namespace scheduled-backup-28431
+ local namespace=scheduled-backup-28431
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces scheduled-backup-28431'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces scheduled-backup-28431
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace scheduled-backup-28431 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.NerqsmIyol
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.mvDTzNhZKk
++ mktemp
+ local LAST_ERR=/tmp/tmp.8ySV0kFziN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.OqFaLxxace
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace scheduled-backup-28431 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NerqsmIyol
+ cat /tmp/tmp.8ySV0kFziN
+ rm /tmp/tmp.NerqsmIyol /tmp/tmp.8ySV0kFziN
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mvDTzNhZKk
+ cat /tmp/tmp.OqFaLxxace
+ rm /tmp/tmp.mvDTzNhZKk /tmp/tmp.OqFaLxxace
+ return 0
+ kubectl_bin wait --for=delete namespace scheduled-backup-28431
++ mktemp
+ local LAST_OUT=/tmp/tmp.mp1RlZGI77
++ mktemp
+ local LAST_ERR=/tmp/tmp.sQ7LhOjjRZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace scheduled-backup-28431
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mp1RlZGI77
+ cat /tmp/tmp.sQ7LhOjjRZ
+ rm /tmp/tmp.mp1RlZGI77 /tmp/tmp.sQ7LhOjjRZ
+ return 0
+ desc 'create namespace scheduled-backup-28431'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace scheduled-backup-28431
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace scheduled-backup-28431
++ mktemp
+ local LAST_OUT=/tmp/tmp.Kb1yO6WkgM
++ mktemp
+ local LAST_ERR=/tmp/tmp.un9CNyP8xm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace scheduled-backup-28431
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kb1yO6WkgM
namespace/scheduled-backup-28431 created
+ cat /tmp/tmp.un9CNyP8xm
+ rm /tmp/tmp.Kb1yO6WkgM /tmp/tmp.un9CNyP8xm
+ return 0
+ set_kube_ctx scheduled-backup-28431
+ local namespace=scheduled-backup-28431
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nRkYDPoOpH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RcxrgjY4yL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nRkYDPoOpH
++ cat /tmp/tmp.RcxrgjY4yL
++ rm /tmp/tmp.nRkYDPoOpH /tmp/tmp.RcxrgjY4yL
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1 --namespace=scheduled-backup-28431
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wk4NwAOIEN
++ mktemp
+ local LAST_ERR=/tmp/tmp.PqRk8cur85
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1 --namespace=scheduled-backup-28431
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Wk4NwAOIEN
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2084-c01795af-5-cluster1" modified.
+ cat /tmp/tmp.PqRk8cur85
+ rm /tmp/tmp.Wk4NwAOIEN /tmp/tmp.PqRk8cur85
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Nov 12 10:41:37 2025
NAMESPACE: scheduled-backup-28431
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.scheduled-backup-28431.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-28431 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-28431

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-28431 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-28431 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2dXA1Jgb8c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KxxPFdGVjW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2dXA1Jgb8c
++ cat /tmp/tmp.KxxPFdGVjW
++ rm /tmp/tmp.2dXA1Jgb8c /tmp/tmp.KxxPFdGVjW
++ return 0
+ MINIO_POD=minio-service-d9589b474-kfbzx
+ wait_pod minio-service-d9589b474-kfbzx
+ local pod=minio-service-d9589b474-kfbzx
+ set +o xtrace
waiting for pod/minio-service-d9589b474-kfbzx to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-28431.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.U1OQseRmXj
++ mktemp
+ local LAST_ERR=/tmp/tmp.GeAyJYEMRp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-28431.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U1OQseRmXj
service/minio-service created
+ cat /tmp/tmp.GeAyJYEMRp
+ rm /tmp/tmp.U1OQseRmXj /tmp/tmp.GeAyJYEMRp
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.s2mU4Enn2t
++ mktemp
+ local LAST_ERR=/tmp/tmp.GMCUsLRsKX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s2mU4Enn2t
make_bucket: operator-testing
pod "aws-cli" deleted from scheduled-backup-28431 namespace
+ cat /tmp/tmp.GMCUsLRsKX
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-28431
+ rm /tmp/tmp.s2mU4Enn2t /tmp/tmp.GMCUsLRsKX
+ return 0
+ label_node
++ kubectl_bin get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
++ head -n1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WMWUk7e1il
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MZayX9TdjC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WMWUk7e1il
++ cat /tmp/tmp.MZayX9TdjC
++ rm /tmp/tmp.WMWUk7e1il /tmp/tmp.MZayX9TdjC
++ return 0
+ LABELED_NODE=gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl
+ desc 'add labels'
+ set +o xtrace


-----------------------------------------------------------------------------------
add labels
-----------------------------------------------------------------------------------

+ kubectl_bin label nodes gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl backupWorker=True --overwrite
++ mktemp
+ local LAST_OUT=/tmp/tmp.gtTJ5IeY75
++ mktemp
+ local LAST_ERR=/tmp/tmp.HP24vMmqmO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label nodes gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl backupWorker=True --overwrite
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gtTJ5IeY75
node/gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl not labeled
+ cat /tmp/tmp.HP24vMmqmO
+ rm /tmp/tmp.gtTJ5IeY75 /tmp/tmp.HP24vMmqmO
+ return 0
+ desc 'create PriorityClass'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PriorityClass
-----------------------------------------------------------------------------------

+ cat -
+ kubectl apply -f -
priorityclass.scheduling.k8s.io/high-priority configured
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.vmI7VdhELa
++ mktemp
+ local LAST_ERR=/tmp/tmp.YXeOvnuhu0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vmI7VdhELa
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.YXeOvnuhu0
+ rm /tmp/tmp.vmI7VdhELa /tmp/tmp.YXeOvnuhu0
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ic3YsAtu1L
++ mktemp
+ local LAST_ERR=/tmp/tmp.PqSlDqtN2B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ic3YsAtu1L
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.PqSlDqtN2B
+ rm /tmp/tmp.Ic3YsAtu1L /tmp/tmp.PqSlDqtN2B
+ return 0
+ cluster=some-name-rs0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2084-c01795af"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.AjBtpD3oWb
++ mktemp
+ local LAST_ERR=/tmp/tmp.7vMI7k1OJM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AjBtpD3oWb
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.7vMI7k1OJM
+ rm /tmp/tmp.AjBtpD3oWb /tmp/tmp.7vMI7k1OJM
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6V5NYqEy9y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RySMJwomNp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6V5NYqEy9y
++ cat /tmp/tmp.RySMJwomNp
++ rm /tmp/tmp.6V5NYqEy9y /tmp/tmp.RySMJwomNp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.adAkh3f6Gx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2MveHvQUOw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.adAkh3f6Gx
++ cat /tmp/tmp.2MveHvQUOw
++ rm /tmp/tmp.adAkh3f6Gx /tmp/tmp.2MveHvQUOw
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EIb2bsrTJx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6sGJZOSGrW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EIb2bsrTJx
++ cat /tmp/tmp.6sGJZOSGrW
++ rm /tmp/tmp.EIb2bsrTJx /tmp/tmp.6sGJZOSGrW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.nzI0kIuSa3/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("scheduled-backup-28431", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7nbwx9teuJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.D0puEkvk8L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7nbwx9teuJ
+ cat /tmp/tmp.D0puEkvk8L
+ rm /tmp/tmp.7nbwx9teuJ /tmp/tmp.D0puEkvk8L
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.nzI0kIuSa3/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.nzI0kIuSa3/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.nzI0kIuSa3/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.nzI0kIuSa3/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-11-12T10:43:43+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-28431
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-28431
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aTQ2VF2c91
+++ mktemp
++ local LAST_ERR=/tmp/tmp.veO2CEdDlR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aTQ2VF2c91
++ cat /tmp/tmp.veO2CEdDlR
++ rm /tmp/tmp.aTQ2VF2c91 /tmp/tmp.veO2CEdDlR
++ return 0
+ local client_container=psmdb-client-696897d69b-lk4bm
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-28431 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.h2giWbWHUQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.lN1yMS1gGi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h2giWbWHUQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("982df6fb-10e5-4d1f-85db-60785f800c5c") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.lN1yMS1gGi
+ rm /tmp/tmp.h2giWbWHUQ /tmp/tmp.lN1yMS1gGi
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.scheduled-backup-28431
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-28431
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DXN7pJC0CY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nANkyrtzvs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DXN7pJC0CY
++ cat /tmp/tmp.nANkyrtzvs
++ rm /tmp/tmp.DXN7pJC0CY /tmp/tmp.nANkyrtzvs
++ return 0
+ local client_container=psmdb-client-696897d69b-lk4bm
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-28431 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XBHGCHCAT4
++ mktemp
+ local LAST_ERR=/tmp/tmp.8FgSk4Rnca
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XBHGCHCAT4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a17878d1-c522-4022-b313-7fa2514748d0") }
Percona Server for MongoDB server version: v8.0.12-4
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.8FgSk4Rnca
+ rm /tmp/tmp.XBHGCHCAT4 /tmp/tmp.8FgSk4Rnca
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-12T10:43:49+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.oxiCril72h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b5OuNdYLcd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oxiCril72h
++ cat /tmp/tmp.b5OuNdYLcd
++ rm /tmp/tmp.oxiCril72h /tmp/tmp.b5OuNdYLcd
++ return 0
+ local client_container=psmdb-client-696897d69b-lk4bm
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kRRBOXgRAN
++ mktemp
+ local LAST_ERR=/tmp/tmp.xLMG7DHHQR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kRRBOXgRAN
+ cat /tmp/tmp.xLMG7DHHQR
+ rm /tmp/tmp.kRRBOXgRAN /tmp/tmp.xLMG7DHHQR
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.nzI0kIuSa3/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-12T10:43:51+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.5Nw5KppdvU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1sq00KTnIV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Nw5KppdvU
++ cat /tmp/tmp.1sq00KTnIV
++ rm /tmp/tmp.5Nw5KppdvU /tmp/tmp.1sq00KTnIV
++ return 0
+ local client_container=psmdb-client-696897d69b-lk4bm
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8LuXGwBWX2
++ mktemp
+ local LAST_ERR=/tmp/tmp.6699tVHQVZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8LuXGwBWX2
+ cat /tmp/tmp.6699tVHQVZ
+ rm /tmp/tmp.8LuXGwBWX2 /tmp/tmp.6699tVHQVZ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.nzI0kIuSa3/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-11-12T10:43:53+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
++ local LAST_OUT=/tmp/tmp.T5xvCDHdAt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ivymks58AF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T5xvCDHdAt
++ cat /tmp/tmp.ivymks58AF
++ rm /tmp/tmp.T5xvCDHdAt /tmp/tmp.ivymks58AF
++ return 0
+ local client_container=psmdb-client-696897d69b-lk4bm
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1fHYHm13yQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.tGLyWohtO9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-lk4bm -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1fHYHm13yQ
+ cat /tmp/tmp.tGLyWohtO9
+ rm /tmp/tmp.1fHYHm13yQ /tmp/tmp.tGLyWohtO9
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.nzI0kIuSa3/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-11-12T10:43:40.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-11-12T10:43:45.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-11-12T10:43:46.000+0000 I listening for the commands

+ desc 'add backups schedule, wait for backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
add backups schedule, wait for backups
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2084-c01795af"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.KqApT3oOWj
++ mktemp
+ local LAST_ERR=/tmp/tmp.0fjDd7Iqrm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KqApT3oOWj
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.0fjDd7Iqrm
+ rm /tmp/tmp.KqApT3oOWj /tmp/tmp.0fjDd7Iqrm
+ return 0
+ sleep 55
+ desc 'disable backups schedule'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable backups schedule
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2084/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2084-c01795af"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.AyzKfi4R8Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.o4JPZ7vBwH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AyzKfi4R8Y
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.o4JPZ7vBwH
+ rm /tmp/tmp.AyzKfi4R8Y /tmp/tmp.o4JPZ7vBwH
+ return 0
+ '[' -z '' ']'
++ kubectl_bin get psmdb-backup
++ grep aws-s3
++ awk '{print$1}'
+++ mktemp
++ head -1
++ local LAST_OUT=/tmp/tmp.dQHDexmzsc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CWHBUIUMoc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dQHDexmzsc
++ cat /tmp/tmp.CWHBUIUMoc
No resources found in scheduled-backup-28431 namespace.
++ rm /tmp/tmp.dQHDexmzsc /tmp/tmp.CWHBUIUMoc
++ return 0
+ backup_name_aws=
++ kubectl_bin get psmdb-backup
++ grep gcp-cs
++ head -1
++ awk '{print$1}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.upBE5nI40L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.emI9LJktR2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.upBE5nI40L
++ cat /tmp/tmp.emI9LJktR2
No resources found in scheduled-backup-28431 namespace.
++ rm /tmp/tmp.upBE5nI40L /tmp/tmp.emI9LJktR2
++ return 0
+ backup_name_gcp=
++ kubectl_bin get psmdb-backup
++ grep azure-blob
++ awk '{print$1}'
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EiyovYzMkn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.heL9yYp5nH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EiyovYzMkn
++ cat /tmp/tmp.heL9yYp5nH
No resources found in scheduled-backup-28431 namespace.
++ rm /tmp/tmp.EiyovYzMkn /tmp/tmp.heL9yYp5nH
++ return 0
+ backup_name_azure=
+ wait_backup ''
+ local backup_name=
+ local target_state=ready
+ set +o xtrace
waiting for  to reach ready state........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................2025-11-12T10:41:05.646Z	INFO	setup	Manager starting up	{"gitCommit": "c01795afa449ea1179a20e20bf96b3936f20ad1e", "gitBranch": "PR-2084-c01795af", "buildTime": "", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
2025-11-12T10:41:05.676Z	INFO	server version	{"platform": "kubernetes", "version": "v1.31.13-gke.1231000"}
2025-11-12T10:41:05.698Z	INFO	controller-runtime.metrics	Starting metrics server
2025-11-12T10:41:05.698Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-11-12T10:41:05.698Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
I1112 10:41:05.698536       1 leaderelection.go:257] attempting to acquire leader lease psmdb-operator/08db0feb.percona.com...
I1112 10:41:05.753676       1 leaderelection.go:271] successfully acquired lease psmdb-operator/08db0feb.percona.com
2025-11-12T10:41:05.753Z	DEBUG	events	percona-server-mongodb-operator-68b58d889-m4lqv_f15b3506-c971-4d04-88d0-308ea2cc63af became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"psmdb-operator","name":"08db0feb.percona.com","uid":"d9c14d51-2fcf-4422-85de-fc309c5ea584","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1762944065744719015"}, "reason": "LeaderElection"}
2025-11-12T10:41:05.754Z	INFO	Starting EventSource	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "source": "kind source: *v1.PerconaServerMongoDB"}
2025-11-12T10:41:05.754Z	INFO	Starting EventSource	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.Pod"}
2025-11-12T10:41:05.754Z	INFO	Starting EventSource	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.Pod"}
2025-11-12T10:41:05.754Z	INFO	Starting EventSource	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "source": "kind source: *v1.PerconaServerMongoDBBackup"}
2025-11-12T10:41:05.754Z	INFO	Starting EventSource	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "source": "kind source: *v1.PerconaServerMongoDBRestore"}
2025-11-12T10:41:06.288Z	INFO	Starting Controller	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB"}
2025-11-12T10:41:06.288Z	INFO	Starting Controller	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup"}
2025-11-12T10:41:06.288Z	INFO	Starting workers	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "worker count": 1}
2025-11-12T10:41:06.288Z	INFO	Starting workers	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "worker count": 1}
2025-11-12T10:41:06.288Z	INFO	Starting Controller	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore"}
2025-11-12T10:41:06.288Z	INFO	Starting workers	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "worker count": 1}
2025-11-12T10:42:23.946Z	INFO	Set CR version	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "version": "1.22.0"}
2025-11-12T10:42:24.836Z	INFO	Created a new mongo key	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "KeyName": "some-name-mongodb-keyfile"}
2025-11-12T10:42:24.863Z	INFO	Created a new mongo key	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "KeyName": "some-name-mongodb-encryption-key"}
2025-11-12T10:42:24.863Z	INFO	Creating or updating backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "name": "weekly", "namespace": "scheduled-backup-28431", "schedule": "0 0 * * 0"}
2025-11-12T10:42:24.910Z	DEBUG	Object created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "name": "some-name-rs0", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}", "generation": 0, "resourceVersion": ""}
2025-11-12T10:42:25.281Z	DEBUG	Object created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "name": "some-name-rs0", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "generation": 0, "resourceVersion": ""}
2025-11-12T10:42:25.319Z	DEBUG	Object created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "name": "some-name-mongod-rs0", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "generation": 0, "resourceVersion": ""}
2025-11-12T10:42:25.685Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "replset": "rs0", "size": 3, "pods": 0}
2025-11-12T10:42:25.685Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:26.232Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:26.232Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "77b47a70-330b-4aa3-b718-7384d423e76d", "previous": "", "current": "initializing"}
2025-11-12T10:42:27.238Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "a4692ccd-2d3f-44a9-befd-f0698bac5983", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:27.238Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "a4692ccd-2d3f-44a9-befd-f0698bac5983", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:27.554Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "a4692ccd-2d3f-44a9-befd-f0698bac5983", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:28.539Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2e76af9f-8587-47c2-81cf-7d766fe37db4", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:28.539Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2e76af9f-8587-47c2-81cf-7d766fe37db4", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:28.855Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2e76af9f-8587-47c2-81cf-7d766fe37db4", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:32.712Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "9c00a945-eff2-4c57-a3a1-f33d4f3bffbe", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:32.712Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "9c00a945-eff2-4c57-a3a1-f33d4f3bffbe", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:32.982Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "9c00a945-eff2-4c57-a3a1-f33d4f3bffbe", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:38.795Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "b549a35c-48c8-4dce-86d5-12b7efa9a75c", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:38.795Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "b549a35c-48c8-4dce-86d5-12b7efa9a75c", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:39.107Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "b549a35c-48c8-4dce-86d5-12b7efa9a75c", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:44.928Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "83ff92a8-ec75-4cc7-aca0-ab1e7e967dbd", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:44.929Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "83ff92a8-ec75-4cc7-aca0-ab1e7e967dbd", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:45.251Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "83ff92a8-ec75-4cc7-aca0-ab1e7e967dbd", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:51.150Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2025647e-1167-4667-a2d7-1730b0e036ff", "replset": "rs0", "size": 3, "pods": 1}
2025-11-12T10:42:51.150Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2025647e-1167-4667-a2d7-1730b0e036ff", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:51.481Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "2025647e-1167-4667-a2d7-1730b0e036ff", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:57.460Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0b4f8eb8-eb62-4ac7-8e0c-fe224b535c8b", "replset": "rs0", "size": 3, "pods": 2}
2025-11-12T10:42:57.460Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0b4f8eb8-eb62-4ac7-8e0c-fe224b535c8b", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:57.790Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0b4f8eb8-eb62-4ac7-8e0c-fe224b535c8b", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:42:58.861Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "754f4e2d-d0b4-407d-bc15-447b8086873e", "replset": "rs0", "size": 3, "pods": 2}
2025-11-12T10:42:58.861Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "754f4e2d-d0b4-407d-bc15-447b8086873e", "replset": "rs0", "status": "initializing"}
2025-11-12T10:42:59.174Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "754f4e2d-d0b4-407d-bc15-447b8086873e", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:43:03.693Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ca59d22a-3753-47d1-83e9-5a9157fcbfae", "replset": "rs0", "size": 3, "pods": 2}
2025-11-12T10:43:03.693Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ca59d22a-3753-47d1-83e9-5a9157fcbfae", "replset": "rs0", "status": "initializing"}
2025-11-12T10:43:04.000Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ca59d22a-3753-47d1-83e9-5a9157fcbfae", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:43:09.900Z	INFO	Waiting for the pods	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0f9f685f-90a3-4424-9480-367d97162140", "replset": "rs0", "size": 3, "pods": 2}
2025-11-12T10:43:09.900Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0f9f685f-90a3-4424-9480-367d97162140", "replset": "rs0", "status": "initializing"}
2025-11-12T10:43:10.224Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "0f9f685f-90a3-4424-9480-367d97162140", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2025-11-12T10:43:26.280Z	INFO	initiating replset	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "replset": "rs0", "pod": "some-name-rs0-0"}
2025-11-12T10:43:29.889Z	INFO	replset initialized	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "replset": "rs0", "pod": "some-name-rs0-0"}
2025-11-12T10:43:34.889Z	INFO	creating user admin	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"}
2025-11-12T10:43:37.026Z	INFO	user admin created	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"}
2025-11-12T10:43:37.177Z	INFO	Creating user	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "database": "admin", "user": "clusterAdmin"}
2025-11-12T10:43:37.234Z	INFO	Creating user	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "database": "admin", "user": "clusterMonitor"}
2025-11-12T10:43:37.323Z	INFO	Creating user	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "database": "admin", "user": "backup"}
2025-11-12T10:43:37.413Z	INFO	Creating user	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "database": "admin", "user": "databaseAdmin"}
2025-11-12T10:43:37.495Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "replset": "rs0", "status": "initializing"}
2025-11-12T10:43:37.782Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "ba0cf3f2-94a2-49bf-a995-bfdeebf7b6a0", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"}
2025-11-12T10:43:39.116Z	INFO	Adding new member to replset	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "_id": 1, "host": "some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017"}
2025-11-12T10:43:39.116Z	INFO	Adding new nodes	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "replset": "rs0"}
2025-11-12T10:43:39.116Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "cfg": {"_id":"rs0","version":2,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-8wrp","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"691464d107c4224251ea5336"},"writeConcernMajorityJournalDefault":true}}
2025-11-12T10:43:39.127Z	INFO	Configuring member votes and priorities	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "replset": "rs0"}
2025-11-12T10:43:39.128Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "cfg": {"_id":"rs0","version":3,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-8wrp","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":0}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"691464d107c4224251ea5336"},"writeConcernMajorityJournalDefault":true}}
2025-11-12T10:43:39.132Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "liveMembers": 1, "pods": 3}
2025-11-12T10:43:39.135Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "replset": "rs0", "status": "initializing"}
2025-11-12T10:43:39.440Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "cd341ab4-2d06-471f-be5f-7ff2c31d7241", "upgradeInProgress": false, "replsetsReady": 1, "clusterState": "initializing"}
2025-11-12T10:43:40.793Z	INFO	Adding new member to replset	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "_id": 2, "host": "some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017"}
2025-11-12T10:43:40.793Z	INFO	Adding new nodes	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "replset": "rs0"}
2025-11-12T10:43:40.793Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "cfg": {"_id":"rs0","version":4,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-8wrp","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":0},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-h1vd","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"691464d107c4224251ea5336"},"writeConcernMajorityJournalDefault":true}}
2025-11-12T10:43:40.807Z	INFO	Configuring member votes and priorities	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "replset": "rs0"}
2025-11-12T10:43:40.807Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "cfg": {"_id":"rs0","version":5,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-8wrp","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-h1vd","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"691464d107c4224251ea5336"},"writeConcernMajorityJournalDefault":true}}
2025-11-12T10:43:40.809Z	ERROR	failed to reconcile cluster	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "replset": "rs0", "error": "failed to update config members: set votes: write mongo config: replSetReconfig: (ConfigurationInProgress) Cannot run replSetReconfig because the node is currently updating its configuration", "errorVerbose": "(ConfigurationInProgress) Cannot run replSetReconfig because the node is currently updating its configuration\nreplSetReconfig\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.(*mongoClient).WriteConfig\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:362\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).updateConfigMembers\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:497\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:242\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:565\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nset votes: write mongo config\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).updateConfigMembers\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:499\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:242\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:565\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to update config members\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:244\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:565\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:567
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:423
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:216
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:461
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296
2025-11-12T10:43:40.814Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "replset": "rs0", "status": "error"}
2025-11-12T10:43:40.873Z	ERROR	Reconciler error	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "35f79d8e-c880-41c6-adfd-37679b854001", "error": "reconcile statefulsets: failed to update config members: set votes: write mongo config: replSetReconfig: (ConfigurationInProgress) Cannot run replSetReconfig because the node is currently updating its configuration", "errorVerbose": "failed to update config members: set votes: write mongo config: replSetReconfig: (ConfigurationInProgress) Cannot run replSetReconfig because the node is currently updating its configuration\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:425\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:474
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.3/pkg/internal/controller/controller.go:296
2025-11-12T10:43:42.150Z	INFO	Configuring member votes and priorities	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "790249c9-01c5-4e05-af17-6c3a811b7a08", "replset": "rs0"}
2025-11-12T10:43:42.150Z	DEBUG	Running replSetReconfig config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "790249c9-01c5-4e05-af17-6c3a811b7a08", "cfg": {"_id":"rs0","version":6,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-8wrp","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-87hl","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.scheduled-backup-28431.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-2084-c0179-default-pool-8e9fec22-h1vd","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"691464d107c4224251ea5336"},"writeConcernMajorityJournalDefault":true}}
2025-11-12T10:43:42.415Z	INFO	Cluster state changed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "790249c9-01c5-4e05-af17-6c3a811b7a08", "previous": "error", "current": "ready"}
2025-11-12T10:43:43.428Z	INFO	update Mongo version to 8.0.12-4 (fetched from db)	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e"}
2025-11-12T10:43:44.560Z	INFO	PBM	pbm-agent version	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "pod": "some-name-rs0-0", "image": "perconalab/percona-server-mongodb-operator:main-backup", "version": "2.12.0"}
2025-11-12T10:43:44.560Z	INFO	PBM	pbm-agent version is different than the go module, this might create problems	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "pbmAgentVersion": "2.12.0", "goModuleVersion": "2.11.0"}
2025-11-12T10:43:44.787Z	INFO	PBM	configuration changed or resync is needed	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "oldHash": "", "newHash": "12280210fecae0f20fcd39b75541bfad249b8e32f5a7c5e12ca53f9897a7889e"}
2025-11-12T10:43:44.801Z	INFO	PBM	Setting config	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "cluster": "some-name", "mainStorage": "aws-s3"}
2025-11-12T10:43:44.835Z	INFO	PBM	Adding profile	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "name": "azure-blob", "type": "azure"}
2025-11-12T10:43:44.855Z	INFO	PBM	Adding profile	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "name": "gcp-cs", "type": "gcs"}
2025-11-12T10:43:44.876Z	INFO	PBM	Adding profile	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "name": "minio", "type": "s3"}
2025-11-12T10:43:44.880Z	INFO	PBM	main storage changed. starting resync	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "91bf181e-2079-48c6-abe6-e3660709d42e", "old": {"type":""}, "new": {"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","bucket":"operator-testing","prefix":"psmdb-scheduled-backup","insecureSkipTLSVerify":false}}}
2025-11-12T10:44:01.300Z	INFO	Creating or updating backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "93baf0f1-673f-42b3-8216-3f7a60dc1721", "name": "every-min-minio", "namespace": "scheduled-backup-28431", "schedule": "* * * * *"}
2025-11-12T10:44:01.301Z	INFO	Creating or updating backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "93baf0f1-673f-42b3-8216-3f7a60dc1721", "name": "every-min-aws-s3", "namespace": "scheduled-backup-28431", "schedule": "* * * * *"}
2025-11-12T10:44:01.301Z	INFO	Creating or updating backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "93baf0f1-673f-42b3-8216-3f7a60dc1721", "name": "every-min-gcp-cs", "namespace": "scheduled-backup-28431", "schedule": "* * * * *"}
2025-11-12T10:44:01.301Z	INFO	Creating or updating backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "93baf0f1-673f-42b3-8216-3f7a60dc1721", "name": "every-min-azure-blob", "namespace": "scheduled-backup-28431", "schedule": "* * * * *"}
2025-11-12T10:44:58.451Z	INFO	deleting outdated backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "c98b15ed-fada-4b9b-a4a5-6e327d15592f", "job": "every-min-gcp-cs"}
2025-11-12T10:44:58.451Z	INFO	deleting outdated backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "c98b15ed-fada-4b9b-a4a5-6e327d15592f", "job": "every-min-aws-s3"}
2025-11-12T10:44:58.451Z	INFO	deleting outdated backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "c98b15ed-fada-4b9b-a4a5-6e327d15592f", "job": "every-min-azure-blob"}
2025-11-12T10:44:58.451Z	INFO	deleting outdated backup job	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"scheduled-backup-28431"}, "namespace": "scheduled-backup-28431", "name": "some-name", "reconcileID": "c98b15ed-fada-4b9b-a4a5-6e327d15592f", "job": "every-min-minio"}
No resources found in scheduled-backup-28431 namespace.
error: arguments in resource/name form must have a single resource and name
error: arguments in resource/name form must have a single resource and name
error: arguments in resource/name form must have a single resource and name
error: arguments in resource/name form must have a single resource and name