Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/non-voting-and-hidden.log
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra non-voting-and-hidden-8747
+ local ns=non-voting-and-hidden-8747
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.XFULDgeMQM
++ mktemp
+ local LAST_ERR=/tmp/tmp.SleHjmch36
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XFULDgeMQM
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.SleHjmch36
+ rm /tmp/tmp.XFULDgeMQM /tmp/tmp.SleHjmch36
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vX64LsTIpE
++ mktemp
+ local LAST_ERR=/tmp/tmp.scnoNgxV9q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vX64LsTIpE
+ cat /tmp/tmp.scnoNgxV9q
+ rm /tmp/tmp.vX64LsTIpE /tmp/tmp.scnoNgxV9q
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.30s9EPl1XX
++ mktemp
+ local LAST_ERR=/tmp/tmp.303UuG2b2N
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.30s9EPl1XX
+ cat /tmp/tmp.303UuG2b2N
+ rm /tmp/tmp.30s9EPl1XX /tmp/tmp.303UuG2b2N
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.DbmQa9F3tN
++ mktemp
+ local LAST_ERR=/tmp/tmp.gYP6M7Zm1M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DbmQa9F3tN
+ cat /tmp/tmp.gYP6M7Zm1M
+ rm /tmp/tmp.DbmQa9F3tN /tmp/tmp.gYP6M7Zm1M
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.MHtWeJQQNi
++ mktemp
+ local LAST_ERR=/tmp/tmp.u6nfFky80r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MHtWeJQQNi
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.u6nfFky80r
+ rm /tmp/tmp.MHtWeJQQNi /tmp/tmp.u6nfFky80r
+ return 0
+ check_crd_for_deletion PR-1961-7f9b8fd6
+ local git_tag=PR-1961-7f9b8fd6
++ yq eval .metadata.name
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-7f9b8fd6/deploy/crd.yaml
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NDKAfmsqGS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7lEUkskDgn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NDKAfmsqGS
++ cat /tmp/tmp.7lEUkskDgn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NDKAfmsqGS
++ cat /tmp/tmp.7lEUkskDgn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NDKAfmsqGS
++ cat /tmp/tmp.7lEUkskDgn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.NDKAfmsqGS
++ cat /tmp/tmp.7lEUkskDgn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.NDKAfmsqGS /tmp/tmp.7lEUkskDgn
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
++ mktemp
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.LygdUELl9y
+ local LAST_OUT=/tmp/tmp.Ic6rSJbwRW
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.KXiWnBs874
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.pbPaiZ0d8J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LygdUELl9y
+ cat /tmp/tmp.KXiWnBs874
+ rm /tmp/tmp.LygdUELl9y /tmp/tmp.KXiWnBs874
+ return 0
namespace "cert-manager" deleted
namespace "non-voting-and-hidden-13969" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ic6rSJbwRW
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.pbPaiZ0d8J
+ rm /tmp/tmp.Ic6rSJbwRW /tmp/tmp.pbPaiZ0d8J
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9TN7sbj1Z8
++ mktemp
+ local LAST_ERR=/tmp/tmp.qvfsktEOtp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9TN7sbj1Z8
+ cat /tmp/tmp.qvfsktEOtp
+ rm /tmp/tmp.9TN7sbj1Z8 /tmp/tmp.qvfsktEOtp
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.l5fMBGVFdx
++ mktemp
+ local LAST_ERR=/tmp/tmp.iQuk5B13zY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l5fMBGVFdx
namespace/psmdb-operator created
+ cat /tmp/tmp.iQuk5B13zY
+ rm /tmp/tmp.l5fMBGVFdx /tmp/tmp.iQuk5B13zY
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rzFihokdqp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yv0jw1U5rp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rzFihokdqp
++ cat /tmp/tmp.Yv0jw1U5rp
++ rm /tmp/tmp.rzFihokdqp /tmp/tmp.Yv0jw1U5rp
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ENBfIYMn3X
++ mktemp
+ local LAST_ERR=/tmp/tmp.2q7fNTRS7v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ENBfIYMn3X
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4" modified.
+ cat /tmp/tmp.2q7fNTRS7v
+ rm /tmp/tmp.ENBfIYMn3X /tmp/tmp.2q7fNTRS7v
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RbZE7hm1Dg
++ mktemp
+ local LAST_ERR=/tmp/tmp.oUwIXVLuMN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RbZE7hm1Dg
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.oUwIXVLuMN
+ rm /tmp/tmp.RbZE7hm1Dg /tmp/tmp.oUwIXVLuMN
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.tKg0zlfHYL
++ mktemp
+ local LAST_ERR=/tmp/tmp.7tcrxPuP0j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tKg0zlfHYL
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.7tcrxPuP0j
+ rm /tmp/tmp.tKg0zlfHYL /tmp/tmp.7tcrxPuP0j
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1UjffkOPzb
++ mktemp
+ local LAST_ERR=/tmp/tmp.wAEr8IWw9O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1UjffkOPzb
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.wAEr8IWw9O
+ rm /tmp/tmp.1UjffkOPzb /tmp/tmp.wAEr8IWw9O
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s2TYlNEcAB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kbnj7IO7WY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s2TYlNEcAB
++ cat /tmp/tmp.kbnj7IO7WY
++ rm /tmp/tmp.s2TYlNEcAB /tmp/tmp.kbnj7IO7WY
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7b94977d54-82fbc
+ local pod=percona-server-mongodb-operator-7b94977d54-82fbc
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7b94977d54-82fbc to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hiLtoBxuFf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XUu8HSvKWl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hiLtoBxuFf
++ cat /tmp/tmp.XUu8HSvKWl
++ rm /tmp/tmp.hiLtoBxuFf /tmp/tmp.XUu8HSvKWl
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7b94977d54-82fbc
++ mktemp
+ local LAST_OUT=/tmp/tmp.ASnYPI10EQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.EcLR4lcboX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7b94977d54-82fbc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ASnYPI10EQ
+ cat /tmp/tmp.EcLR4lcboX
+ rm /tmp/tmp.ASnYPI10EQ /tmp/tmp.EcLR4lcboX
+ return 0
2026-01-12T22:50:16.279Z	INFO	setup	Manager starting up	{"gitCommit": "7f9b8fd660ef537a8ff13f3505cffba918719e96", "gitBranch": "PR-1961-7f9b8fd6", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace non-voting-and-hidden-8747
+ local namespace=non-voting-and-hidden-8747
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces non-voting-and-hidden-8747'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces non-voting-and-hidden-8747
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace non-voting-and-hidden-8747 --ignore-not-found
++ mktemp
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.I1Pw60oWEF
++ mktemp
+ local LAST_OUT=/tmp/tmp.x6ebU0gg2a
+ local LAST_ERR=/tmp/tmp.4sDgXVr2T1
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.E3t5XOa6dV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace non-voting-and-hidden-8747 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I1Pw60oWEF
+ cat /tmp/tmp.4sDgXVr2T1
+ rm /tmp/tmp.I1Pw60oWEF /tmp/tmp.4sDgXVr2T1
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x6ebU0gg2a
+ cat /tmp/tmp.E3t5XOa6dV
+ rm /tmp/tmp.x6ebU0gg2a /tmp/tmp.E3t5XOa6dV
+ return 0
+ kubectl_bin wait --for=delete namespace non-voting-and-hidden-8747
++ mktemp
+ local LAST_OUT=/tmp/tmp.JiVBf1YwzK
++ mktemp
+ local LAST_ERR=/tmp/tmp.m2SqfVi64T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace non-voting-and-hidden-8747
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JiVBf1YwzK
+ cat /tmp/tmp.m2SqfVi64T
+ rm /tmp/tmp.JiVBf1YwzK /tmp/tmp.m2SqfVi64T
+ return 0
+ desc 'create namespace non-voting-and-hidden-8747'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace non-voting-and-hidden-8747
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace non-voting-and-hidden-8747
++ mktemp
+ local LAST_OUT=/tmp/tmp.HPdlJqgTQD
++ mktemp
+ local LAST_ERR=/tmp/tmp.GrTrjeRh9S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace non-voting-and-hidden-8747
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HPdlJqgTQD
namespace/non-voting-and-hidden-8747 created
+ cat /tmp/tmp.GrTrjeRh9S
+ rm /tmp/tmp.HPdlJqgTQD /tmp/tmp.GrTrjeRh9S
+ return 0
+ set_kube_ctx non-voting-and-hidden-8747
+ local namespace=non-voting-and-hidden-8747
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jP4zNX7QFM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LwtKox9Dht
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jP4zNX7QFM
++ cat /tmp/tmp.LwtKox9Dht
++ rm /tmp/tmp.jP4zNX7QFM /tmp/tmp.LwtKox9Dht
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4 --namespace=non-voting-and-hidden-8747
++ mktemp
+ local LAST_OUT=/tmp/tmp.GMxKmHIKZg
++ mktemp
+ local LAST_ERR=/tmp/tmp.Au5aQSGFCF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4 --namespace=non-voting-and-hidden-8747
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GMxKmHIKZg
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster4" modified.
+ cat /tmp/tmp.Au5aQSGFCF
+ rm /tmp/tmp.GMxKmHIKZg /tmp/tmp.Au5aQSGFCF
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.O5fqZAlR7G
++ mktemp
+ local LAST_ERR=/tmp/tmp.1aQfu53WVP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.O5fqZAlR7G
namespace/cert-manager created
+ cat /tmp/tmp.1aQfu53WVP
+ rm /tmp/tmp.O5fqZAlR7G /tmp/tmp.1aQfu53WVP
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.1IfHj7i2zS
++ mktemp
+ local LAST_ERR=/tmp/tmp.5K15XCImdA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1IfHj7i2zS
namespace/cert-manager labeled
+ cat /tmp/tmp.5K15XCImdA
+ rm /tmp/tmp.1IfHj7i2zS /tmp/tmp.5K15XCImdA
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.MzJnZNq39N
++ mktemp
+ local LAST_ERR=/tmp/tmp.LYZQfsrNOW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MzJnZNq39N
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.LYZQfsrNOW
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.MzJnZNq39N /tmp/tmp.LYZQfsrNOW
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.7oeZcZHyIh
++ mktemp
+ local LAST_ERR=/tmp/tmp.i0QFOkDCpN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7oeZcZHyIh
pod/cert-manager-cainjector-5dc9c8b4f7-kfkh9 condition met
pod/cert-manager-df4b69479-q2ng7 condition met
pod/cert-manager-webhook-769bbb594d-jv9ms condition met
+ cat /tmp/tmp.i0QFOkDCpN
+ rm /tmp/tmp.7oeZcZHyIh /tmp/tmp.i0QFOkDCpN
+ return 0
+ sleep 120
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.f4YgMlOqou
++ mktemp
+ local LAST_ERR=/tmp/tmp.IIeDJ6DgxK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f4YgMlOqou
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.IIeDJ6DgxK
+ rm /tmp/tmp.f4YgMlOqou /tmp/tmp.IIeDJ6DgxK
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Jan 12 22:53:19 2026
NAMESPACE: non-voting-and-hidden-8747
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.non-voting-and-hidden-8747.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace non-voting-and-hidden-8747 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace non-voting-and-hidden-8747

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace non-voting-and-hidden-8747 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace non-voting-and-hidden-8747 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dCQUknWuRH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TFRfnAlma7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dCQUknWuRH
++ cat /tmp/tmp.TFRfnAlma7
++ rm /tmp/tmp.dCQUknWuRH /tmp/tmp.TFRfnAlma7
++ return 0
+ MINIO_POD=minio-service-d9589b474-wstmd
+ wait_pod minio-service-d9589b474-wstmd
+ local pod=minio-service-d9589b474-wstmd
+ set +o xtrace
waiting for pod/minio-service-d9589b474-wstmd to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.non-voting-and-hidden-8747.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.vP34ZqQ1IY
++ mktemp
+ local LAST_ERR=/tmp/tmp.bt0utiF6R5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.non-voting-and-hidden-8747.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vP34ZqQ1IY
service/minio-service created
+ cat /tmp/tmp.bt0utiF6R5
+ rm /tmp/tmp.vP34ZqQ1IY /tmp/tmp.bt0utiF6R5
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.pNtETxCAy1
++ mktemp
+ local LAST_ERR=/tmp/tmp.jRqW3WSk2a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pNtETxCAy1
make_bucket: operator-testing
pod "aws-cli" deleted from non-voting-and-hidden-8747 namespace
+ cat /tmp/tmp.jRqW3WSk2a
+ rm /tmp/tmp.pNtETxCAy1 /tmp/tmp.jRqW3WSk2a
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OTcEncIZwc
++ mktemp
+ local LAST_ERR=/tmp/tmp.jxQ0FJa4Iy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OTcEncIZwc
deployment.apps/psmdb-client created
secret/some-users created
+ cat /tmp/tmp.jxQ0FJa4Iy
+ rm /tmp/tmp.OTcEncIZwc /tmp/tmp.jxQ0FJa4Iy
+ return 0
+ desc 'Creating PSMDB cluster with non-voting + hidden members'
+ set +o xtrace


-----------------------------------------------------------------------------------
Creating PSMDB cluster with non-voting + hidden members
-----------------------------------------------------------------------------------

+ log 'create PSMDB cluster some-name'
+ set +o xtrace
[2026-01-12T22:53:56+0000] create PSMDB cluster some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/non-voting-and-hidden-8747/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.Fv1Jh2Ebw4
++ mktemp
+ local LAST_ERR=/tmp/tmp.PS3uKeRQ1n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Fv1Jh2Ebw4
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.PS3uKeRQ1n
+ rm /tmp/tmp.Fv1Jh2Ebw4 /tmp/tmp.PS3uKeRQ1n
+ return 0
+ log 'check if Pods are started'
+ set +o xtrace
[2026-01-12T22:53:58+0000] check if Pods are started
+ wait_for_running some-name-rs0 3 false
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N6Il7e4NzT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1XNKX7sSII
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N6Il7e4NzT
++ cat /tmp/tmp.1XNKX7sSII
++ rm /tmp/tmp.N6Il7e4NzT /tmp/tmp.1XNKX7sSII
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5H5NbDPu2V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k4OlW16lvk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5H5NbDPu2V
++ cat /tmp/tmp.k4OlW16lvk
++ rm /tmp/tmp.5H5NbDPu2V /tmp/tmp.k4OlW16lvk
++ return 0
+ [[ true == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zPzxvGnZeG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l2Qp6KFDZ7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zPzxvGnZeG
++ cat /tmp/tmp.l2Qp6KFDZ7
++ rm /tmp/tmp.zPzxvGnZeG /tmp/tmp.l2Qp6KFDZ7
++ return 0
+ last_pod=0
++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-nv-0
+ local pod=some-name-rs0-nv-0
+ set +o xtrace
waiting for pod/some-name-rs0-nv-0 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LQJtbgx1oG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AKQE1iUhjd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LQJtbgx1oG
++ cat /tmp/tmp.AKQE1iUhjd
++ rm /tmp/tmp.LQJtbgx1oG /tmp/tmp.AKQE1iUhjd
++ return 0
+ [[ true == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WuZ5GOI3rf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rHz47E4uRb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WuZ5GOI3rf
++ cat /tmp/tmp.rHz47E4uRb
++ rm /tmp/tmp.WuZ5GOI3rf /tmp/tmp.rHz47E4uRb
++ return 0
+ last_pod=1
++ seq 0 1
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-hidden-0
+ local pod=some-name-rs0-hidden-0
+ set +o xtrace
waiting for pod/some-name-rs0-hidden-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-hidden-1
+ local pod=some-name-rs0-hidden-1
+ set +o xtrace
waiting for pod/some-name-rs0-hidden-1 to be ready.OK
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_cluster_state some-name ready
+ local cluster_name=some-name
+ local target_state=ready
+ echo -n 'Waiting for psmdb/some-name to reach ready state'
Waiting for psmdb/some-name to reach ready state+ local timeout=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.INJHKIW9Z3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LHkX84P2EP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.INJHKIW9Z3
++ cat /tmp/tmp.LHkX84P2EP
++ rm /tmp/tmp.INJHKIW9Z3 /tmp/tmp.LHkX84P2EP
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=1
+ echo -n .
.+ [[ 1 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.81xM6qbnFh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.woxBeodglx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.81xM6qbnFh
++ cat /tmp/tmp.woxBeodglx
++ rm /tmp/tmp.81xM6qbnFh /tmp/tmp.woxBeodglx
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=2
+ echo -n .
.+ [[ 2 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YD7Ybua0ZD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bBKbnsXfZW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YD7Ybua0ZD
++ cat /tmp/tmp.bBKbnsXfZW
++ rm /tmp/tmp.YD7Ybua0ZD /tmp/tmp.bBKbnsXfZW
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=3
+ echo -n .
.+ [[ 3 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.viqAlrorFz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pz4OjbvGCP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.viqAlrorFz
++ cat /tmp/tmp.pz4OjbvGCP
++ rm /tmp/tmp.viqAlrorFz /tmp/tmp.pz4OjbvGCP
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=4
+ echo -n .
.+ [[ 4 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RlznQIsP6b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WNDWJYLyL0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RlznQIsP6b
++ cat /tmp/tmp.WNDWJYLyL0
++ rm /tmp/tmp.RlznQIsP6b /tmp/tmp.WNDWJYLyL0
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=5
+ echo -n .
.+ [[ 5 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PwSbyyqMPe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IWfhDaQJYE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PwSbyyqMPe
++ cat /tmp/tmp.IWfhDaQJYE
++ rm /tmp/tmp.PwSbyyqMPe /tmp/tmp.IWfhDaQJYE
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=6
+ echo -n .
.+ [[ 6 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q7xkvGdik1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.95xMQ9OMN6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q7xkvGdik1
++ cat /tmp/tmp.95xMQ9OMN6
++ rm /tmp/tmp.q7xkvGdik1 /tmp/tmp.95xMQ9OMN6
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=7
+ echo -n .
.+ [[ 7 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1Dw1ysJa31
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5EhyqokzXD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1Dw1ysJa31
++ cat /tmp/tmp.5EhyqokzXD
++ rm /tmp/tmp.1Dw1ysJa31 /tmp/tmp.5EhyqokzXD
++ return 0
+ [[ error =~ ready ]]
+ sleep 1
+ timeout=8
+ echo -n .
.+ [[ 8 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.508nuM8Ods
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SPc9hapkas
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.508nuM8Ods
++ cat /tmp/tmp.SPc9hapkas
++ rm /tmp/tmp.508nuM8Ods /tmp/tmp.SPc9hapkas
++ return 0
+ [[ error =~ ready ]]
+ sleep 1
+ timeout=9
+ echo -n .
.+ [[ 9 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QolJfZQcnX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CQfi6UHtys
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QolJfZQcnX
++ cat /tmp/tmp.CQfi6UHtys
++ rm /tmp/tmp.QolJfZQcnX /tmp/tmp.CQfi6UHtys
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=10
+ echo -n .
.+ [[ 10 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1BeLCE4sxf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dqZmyBi9c9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1BeLCE4sxf
++ cat /tmp/tmp.dqZmyBi9c9
++ rm /tmp/tmp.1BeLCE4sxf /tmp/tmp.dqZmyBi9c9
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=11
+ echo -n .
.+ [[ 11 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tkvy3DsSx5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kLoj7lTSlI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tkvy3DsSx5
++ cat /tmp/tmp.kLoj7lTSlI
++ rm /tmp/tmp.Tkvy3DsSx5 /tmp/tmp.kLoj7lTSlI
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=12
+ echo -n .
.+ [[ 12 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.evK2at8f7v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5mHtKxtl0P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.evK2at8f7v
++ cat /tmp/tmp.5mHtKxtl0P
++ rm /tmp/tmp.evK2at8f7v /tmp/tmp.5mHtKxtl0P
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=13
+ echo -n .
.+ [[ 13 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iKfSPkpe6j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0h0XayXrv9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iKfSPkpe6j
++ cat /tmp/tmp.0h0XayXrv9
++ rm /tmp/tmp.iKfSPkpe6j /tmp/tmp.0h0XayXrv9
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=14
+ echo -n .
.+ [[ 14 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2WTUP4i5Zd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LYKTdyvMBz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2WTUP4i5Zd
++ cat /tmp/tmp.LYKTdyvMBz
++ rm /tmp/tmp.2WTUP4i5Zd /tmp/tmp.LYKTdyvMBz
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=15
+ echo -n .
.+ [[ 15 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YQVPnVK6Fb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IFLWJg8adt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YQVPnVK6Fb
++ cat /tmp/tmp.IFLWJg8adt
++ rm /tmp/tmp.YQVPnVK6Fb /tmp/tmp.IFLWJg8adt
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=16
+ echo -n .
.+ [[ 16 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rQ8xxoanwL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8sl1xVoaKt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rQ8xxoanwL
++ cat /tmp/tmp.8sl1xVoaKt
++ rm /tmp/tmp.rQ8xxoanwL /tmp/tmp.8sl1xVoaKt
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=17
+ echo -n .
.+ [[ 17 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Arf09v5lPo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xcLowHKp2B
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Arf09v5lPo
++ cat /tmp/tmp.xcLowHKp2B
++ rm /tmp/tmp.Arf09v5lPo /tmp/tmp.xcLowHKp2B
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=18
+ echo -n .
.+ [[ 18 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ue5VAW9OZ5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j8N3sh2Irj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ue5VAW9OZ5
++ cat /tmp/tmp.j8N3sh2Irj
++ rm /tmp/tmp.ue5VAW9OZ5 /tmp/tmp.j8N3sh2Irj
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=19
+ echo -n .
.+ [[ 19 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IRsHXSYSv9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yq0ym7A4ia
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IRsHXSYSv9
++ cat /tmp/tmp.yq0ym7A4ia
++ rm /tmp/tmp.IRsHXSYSv9 /tmp/tmp.yq0ym7A4ia
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=20
+ echo -n .
.+ [[ 20 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gSBrGyCb5T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eT6K6RO4i5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gSBrGyCb5T
++ cat /tmp/tmp.eT6K6RO4i5
++ rm /tmp/tmp.gSBrGyCb5T /tmp/tmp.eT6K6RO4i5
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=21
+ echo -n .
.+ [[ 21 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D065xMC13D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uewCkIz2uV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D065xMC13D
++ cat /tmp/tmp.uewCkIz2uV
++ rm /tmp/tmp.D065xMC13D /tmp/tmp.uewCkIz2uV
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=22
+ echo -n .
.+ [[ 22 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bW0rraPfz7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W177eXLikS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bW0rraPfz7
++ cat /tmp/tmp.W177eXLikS
++ rm /tmp/tmp.bW0rraPfz7 /tmp/tmp.W177eXLikS
++ return 0
+ [[ ready =~ ready ]]
+ echo

+ log 'psmdb/some-name is ready: OK'
+ set +o xtrace
[2026-01-12T22:56:19+0000] psmdb/some-name is ready: OK
+ log 'check if statefulsets created with expected config'
+ set +o xtrace
[2026-01-12T22:56:19+0000] check if statefulsets created with expected config
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-8747", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xAiobLkXay
++ mktemp
+ local LAST_ERR=/tmp/tmp.meKrcat8FL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xAiobLkXay
+ cat /tmp/tmp.meKrcat8FL
+ rm /tmp/tmp.xAiobLkXay /tmp/tmp.meKrcat8FL
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-01-12T22:56:21+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-rs0-nv
+ local resource=statefulset/some-name-rs0-nv
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml
+ local new_result=/tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-nv.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0-nv
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-8747", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.cmNN1Ir0dz
++ mktemp
+ local LAST_ERR=/tmp/tmp.rd7nXzfMYr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0-nv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cmNN1Ir0dz
+ cat /tmp/tmp.rd7nXzfMYr
+ rm /tmp/tmp.cmNN1Ir0dz /tmp/tmp.rd7nXzfMYr
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-nv.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-nv.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-nv.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-nv.yml
+ log 'compare_kubectl: statefulset/some-name-rs0-nv OK'
+ set +o xtrace
[2026-01-12T22:56:22+0000] compare_kubectl: statefulset/some-name-rs0-nv OK
+ compare_kubectl statefulset/some-name-rs0-hidden
+ local resource=statefulset/some-name-rs0-hidden
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml
+ local new_result=/tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-hidden.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0-hidden
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-8747", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.kTr6cgBEBD
++ mktemp
+ local LAST_ERR=/tmp/tmp.3l5EA6CIAL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0-hidden
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kTr6cgBEBD
+ cat /tmp/tmp.3l5EA6CIAL
+ rm /tmp/tmp.kTr6cgBEBD /tmp/tmp.3l5EA6CIAL
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-hidden.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-hidden.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-hidden.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml /tmp/tmp.MRN7hnsSd5/statefulset_some-name-rs0-hidden.yml
+ log 'compare_kubectl: statefulset/some-name-rs0-hidden OK'
+ set +o xtrace
[2026-01-12T22:56:23+0000] compare_kubectl: statefulset/some-name-rs0-hidden OK
+ log 'check if rs.config is correct'
+ set +o xtrace
[2026-01-12T22:56:23+0000] check if rs.config is correct
+ compare_rs_config 'rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 nonvoting
+ local 'command=rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local compare_file=nonvoting
+ run_mongo 'rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 mongodb ''
+ local 'command=rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1GS4hxyXgP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zT6nWUXG6M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1GS4hxyXgP
++ cat /tmp/tmp.zT6nWUXG6M
++ rm /tmp/tmp.1GS4hxyXgP /tmp/tmp.zT6nWUXG6M
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pnFAKH3Qa9
++ mktemp
+ local LAST_ERR=/tmp/tmp.ykGG5ypJK2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pnFAKH3Qa9
+ cat /tmp/tmp.ykGG5ypJK2
+ rm /tmp/tmp.pnFAKH3Qa9 /tmp/tmp.ykGG5ypJK2
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/nonvoting.json /tmp/tmp.MRN7hnsSd5/nonvoting
+ log 'nonVoting members: OK'
+ set +o xtrace
[2026-01-12T22:56:25+0000] nonVoting members: OK
+ compare_rs_config 'rs.config().members.map(m => m.hidden)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 hidden
+ local 'command=rs.config().members.map(m => m.hidden)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local compare_file=hidden
+ run_mongo 'rs.config().members.map(m => m.hidden)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 mongodb ''
+ local 'command=rs.config().members.map(m => m.hidden)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FwHzRSUpbS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4KQ94BDCAc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FwHzRSUpbS
++ cat /tmp/tmp.4KQ94BDCAc
++ rm /tmp/tmp.FwHzRSUpbS /tmp/tmp.4KQ94BDCAc
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''rs.config().members.map(m => m.hidden)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.OZb6thahrv
++ mktemp
+ local LAST_ERR=/tmp/tmp.m8UVs3yTa5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''rs.config().members.map(m => m.hidden)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OZb6thahrv
+ cat /tmp/tmp.m8UVs3yTa5
+ rm /tmp/tmp.OZb6thahrv /tmp/tmp.m8UVs3yTa5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/hidden.json /tmp/tmp.MRN7hnsSd5/hidden
+ log 'hidden members: OK'
+ set +o xtrace
[2026-01-12T22:56:28+0000] hidden members: OK
+ log 'create user myApp'
+ set +o xtrace
[2026-01-12T22:56:28+0000] create user myApp
+ run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local 'command=db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8uBaDPI0j2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XJgyV7oAvo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8uBaDPI0j2
++ cat /tmp/tmp.XJgyV7oAvo
++ rm /tmp/tmp.8uBaDPI0j2 /tmp/tmp.XJgyV7oAvo
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HuLJN9cQzY
++ mktemp
+ local LAST_ERR=/tmp/tmp.v3MBtuUcLB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HuLJN9cQzY
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("491eb35d-c4b0-42b6-b3bd-84264a056f8b") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.v3MBtuUcLB
+ rm /tmp/tmp.HuLJN9cQzY /tmp/tmp.v3MBtuUcLB
+ return 0
+ log 'write data'
+ set +o xtrace
[2026-01-12T22:56:31+0000] write data
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JXcLlUT2vc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iMTWVa2vlt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JXcLlUT2vc
++ cat /tmp/tmp.iMTWVa2vlt
++ rm /tmp/tmp.JXcLlUT2vc /tmp/tmp.iMTWVa2vlt
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.x4M24i3t5F
++ mktemp
+ local LAST_ERR=/tmp/tmp.zkqhaMYh2u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x4M24i3t5F
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("cfdf11c4-7275-46a7-a37f-e9b94aca040d") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.zkqhaMYh2u
+ rm /tmp/tmp.x4M24i3t5F /tmp/tmp.zkqhaMYh2u
+ return 0
+ sleep 10
+ log 'compare data'
+ set +o xtrace
[2026-01-12T22:56:43+0000] compare data
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:56:43+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.h45EaOGhHG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qd56lSWmXN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h45EaOGhHG
++ cat /tmp/tmp.qd56lSWmXN
++ rm /tmp/tmp.h45EaOGhHG /tmp/tmp.qd56lSWmXN
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mzPegG5FCj
++ mktemp
+ local LAST_ERR=/tmp/tmp.tCFbkuw4nv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mzPegG5FCj
+ cat /tmp/tmp.tCFbkuw4nv
+ rm /tmp/tmp.mzPegG5FCj /tmp/tmp.tCFbkuw4nv
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:56:45+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.njt9x88juS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IIPwtxk9IN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.njt9x88juS
++ cat /tmp/tmp.IIPwtxk9IN
++ rm /tmp/tmp.njt9x88juS /tmp/tmp.IIPwtxk9IN
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7K9bgnvUxm
++ mktemp
+ local LAST_ERR=/tmp/tmp.2VwO2yu2RM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7K9bgnvUxm
+ cat /tmp/tmp.2VwO2yu2RM
+ rm /tmp/tmp.7K9bgnvUxm /tmp/tmp.2VwO2yu2RM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:56:48+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xmn5l7MlZu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Gy71xHhz0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xmn5l7MlZu
++ cat /tmp/tmp.4Gy71xHhz0
++ rm /tmp/tmp.Xmn5l7MlZu /tmp/tmp.4Gy71xHhz0
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7rnMmxJubC
++ mktemp
+ local LAST_ERR=/tmp/tmp.RLIX38re5T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7rnMmxJubC
+ cat /tmp/tmp.RLIX38re5T
+ rm /tmp/tmp.7rnMmxJubC /tmp/tmp.RLIX38re5T
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:56:51+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UPs1WNGT7d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C78dY4QwnW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UPs1WNGT7d
++ cat /tmp/tmp.C78dY4QwnW
++ rm /tmp/tmp.UPs1WNGT7d /tmp/tmp.C78dY4QwnW
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3ZSKL40T5M
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZjTmkb0I6Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3ZSKL40T5M
+ cat /tmp/tmp.ZjTmkb0I6Q
+ rm /tmp/tmp.3ZSKL40T5M /tmp/tmp.ZjTmkb0I6Q
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ run_backup minio backup-logical logical
+ local storage=minio
+ local backup_name=backup-logical
+ local type=logical
+ log 'running backup backup-logical'
+ set +o xtrace
[2026-01-12T22:56:53+0000] running backup backup-logical
+ yq eval '.metadata.name = "backup-logical"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.anpsXU8H3i
++ mktemp
+ local LAST_ERR=/tmp/tmp.suxmIGTJeD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.anpsXU8H3i
perconaservermongodbbackup.psmdb.percona.com/backup-logical created
+ cat /tmp/tmp.suxmIGTJeD
+ rm /tmp/tmp.anpsXU8H3i /tmp/tmp.suxmIGTJeD
+ return 0
+ wait_backup backup-logical ready
+ local backup_name=backup-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-logical to reach ready state.......OK
+ log 'drop collection'
+ set +o xtrace
[2026-01-12T22:57:07+0000] drop collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ERLHBdaHzm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UZ2vEnkKYF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ERLHBdaHzm
++ cat /tmp/tmp.UZ2vEnkKYF
++ rm /tmp/tmp.ERLHBdaHzm /tmp/tmp.UZ2vEnkKYF
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.QxfwuQFdk5
++ mktemp
+ local LAST_ERR=/tmp/tmp.3PkJRFHq99
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QxfwuQFdk5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("1ca65dec-6eae-4a0f-957c-51603b1931c4") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.3PkJRFHq99
+ rm /tmp/tmp.QxfwuQFdk5 /tmp/tmp.3PkJRFHq99
+ return 0
+ run_restore backup-logical
+ local backup_name=backup-logical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ log 'running restore restore-backup-logical'
+ set +o xtrace
[2026-01-12T22:57:10+0000] running restore restore-backup-logical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-logical/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-logical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.HvIwomftWB
++ mktemp
+ local LAST_ERR=/tmp/tmp.mSGoxdyWes
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HvIwomftWB
perconaservermongodbrestore.psmdb.percona.com/restore-backup-logical created
+ cat /tmp/tmp.mSGoxdyWes
+ rm /tmp/tmp.HvIwomftWB /tmp/tmp.mSGoxdyWes
+ return 0
+ wait_restore backup-logical some-name
+ local backup_name=backup-logical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-logical object to be created.OK
Waiting psmdb-restore/restore-backup-logical to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gbiNf0vGEj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IkjWEnPgZr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gbiNf0vGEj
++ cat /tmp/tmp.IkjWEnPgZr
++ rm /tmp/tmp.gbiNf0vGEj /tmp/tmp.IkjWEnPgZr
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ log 'compare data after restore-backup-logical'
+ set +o xtrace
[2026-01-12T22:57:31+0000] compare data after restore-backup-logical
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:57:31+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JPNxENXUnW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QrtK7j7lPS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JPNxENXUnW
++ cat /tmp/tmp.QrtK7j7lPS
++ rm /tmp/tmp.JPNxENXUnW /tmp/tmp.QrtK7j7lPS
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hpf4N1Vj51
++ mktemp
+ local LAST_ERR=/tmp/tmp.KYpyBnaF5T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hpf4N1Vj51
+ cat /tmp/tmp.KYpyBnaF5T
+ rm /tmp/tmp.hpf4N1Vj51 /tmp/tmp.KYpyBnaF5T
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:57:34+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ [[ myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rZtFCd6WFY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZFhyV3HUwt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rZtFCd6WFY
++ cat /tmp/tmp.ZFhyV3HUwt
++ rm /tmp/tmp.rZtFCd6WFY /tmp/tmp.ZFhyV3HUwt
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.koa5sy7Oep
++ mktemp
+ local LAST_ERR=/tmp/tmp.pnui1htMZH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.koa5sy7Oep
+ cat /tmp/tmp.pnui1htMZH
+ rm /tmp/tmp.koa5sy7Oep /tmp/tmp.pnui1htMZH
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:57:37+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c1XzyLzaIz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oRpYJltED4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c1XzyLzaIz
++ cat /tmp/tmp.oRpYJltED4
++ rm /tmp/tmp.c1XzyLzaIz /tmp/tmp.oRpYJltED4
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ir3wzyh7Wb
++ mktemp
+ local LAST_ERR=/tmp/tmp.8oQNY0zrSO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ir3wzyh7Wb
+ cat /tmp/tmp.8oQNY0zrSO
+ rm /tmp/tmp.ir3wzyh7Wb /tmp/tmp.8oQNY0zrSO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-01-12T22:57:39+0000] running db.test.find() in myApp
+ [[ false == \t\r\u\e ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TRnEmmBuwg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QRSSQRSkfp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TRnEmmBuwg
++ cat /tmp/tmp.QRSSQRSkfp
++ rm /tmp/tmp.TRnEmmBuwg /tmp/tmp.QRSSQRSkfp
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qEVJDN5b7y
++ mktemp
+ local LAST_ERR=/tmp/tmp.IKKjmzGCRN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qEVJDN5b7y
+ cat /tmp/tmp.IKKjmzGCRN
+ rm /tmp/tmp.qEVJDN5b7y /tmp/tmp.IKKjmzGCRN
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.MRN7hnsSd5/find
+ run_backup minio backup-physical physical
+ local storage=minio
+ local backup_name=backup-physical
+ local type=physical
+ log 'running backup backup-physical'
+ set +o xtrace
[2026-01-12T22:57:41+0000] running backup backup-physical
+ yq eval '.metadata.name = "backup-physical"
			| .spec.storageName = "minio"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.I1Ta3TBXOn
++ mktemp
+ local LAST_ERR=/tmp/tmp.LrhlXNeKMd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I1Ta3TBXOn
perconaservermongodbbackup.psmdb.percona.com/backup-physical created
+ cat /tmp/tmp.LrhlXNeKMd
+ rm /tmp/tmp.I1Ta3TBXOn /tmp/tmp.LrhlXNeKMd
+ return 0
+ wait_backup backup-physical ready
+ local backup_name=backup-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-physical to reach ready state.......OK
+ log 'drop collection'
+ set +o xtrace
[2026-01-12T22:57:56+0000] drop collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-8747
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-8747 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l3WklCoEop
+++ mktemp
++ local LAST_ERR=/tmp/tmp.feOd9mthJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l3WklCoEop
++ cat /tmp/tmp.feOd9mthJ0
++ rm /tmp/tmp.l3WklCoEop /tmp/tmp.feOd9mthJ0
++ return 0
+ local client_container=psmdb-client-696897d69b-5924x
+ kubectl_bin exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dvicqprp92
++ mktemp
+ local LAST_ERR=/tmp/tmp.g1DuoxO7tW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5924x -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dvicqprp92
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7b0a0acd-4872-4e68-adc0-d97cc364cd4d") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.g1DuoxO7tW
+ rm /tmp/tmp.dvicqprp92 /tmp/tmp.g1DuoxO7tW
+ return 0
+ run_restore backup-physical
+ local backup_name=backup-physical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ log 'running restore restore-backup-physical'
+ set +o xtrace
[2026-01-12T22:57:58+0000] running restore restore-backup-physical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-physical/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.InEROtgf3j
++ mktemp
+ local LAST_ERR=/tmp/tmp.CDtk3ywSAn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.InEROtgf3j
perconaservermongodbrestore.psmdb.percona.com/restore-backup-physical created
+ cat /tmp/tmp.CDtk3ywSAn
+ rm /tmp/tmp.InEROtgf3j /tmp/tmp.CDtk3ywSAn
+ return 0
+ wait_restore backup-physical some-name
+ local backup_name=backup-physical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-physical object to be created.OK
Waiting psmdb-restore/restore-backup-physical to reach state "ready" .............................

-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-0
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.Reb1xLKzPZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.7gZKxtR3pe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Reb1xLKzPZ
2026/01/12 23:01:29 [entrypoint] starting `pbm-agent`
2026-01-12T23:01:31.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2026-01-12T23:01:31.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-01-12T23:01:31.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2026-01-12T23:01:31.000+0000 I starting PITR routine
2026-01-12T23:01:31.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017
2026-01-12T23:01:31.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-01-12T23:01:31.000+0000 I listening for the commands
+ cat /tmp/tmp.7gZKxtR3pe
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.Reb1xLKzPZ /tmp/tmp.7gZKxtR3pe
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-1
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.dFz5XCaqPJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.OrKVi2qaWT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dFz5XCaqPJ
2026/01/12 23:00:54 [entrypoint] starting `pbm-agent`
2026-01-12T23:00:56.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2026-01-12T23:00:56.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-01-12T23:00:56.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2026-01-12T23:00:56.000+0000 I starting PITR routine
2026-01-12T23:00:56.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017
2026-01-12T23:00:56.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-01-12T23:00:56.000+0000 I listening for the commands
+ cat /tmp/tmp.OrKVi2qaWT
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.dFz5XCaqPJ /tmp/tmp.OrKVi2qaWT
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-2
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.EwJIUI7Qwa
++ mktemp
+ local LAST_ERR=/tmp/tmp.QQqqhebuJU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EwJIUI7Qwa
2026/01/12 23:00:20 [entrypoint] starting `pbm-agent`
2026-01-12T23:00:22.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb


2026-01-12T23:00:22.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-01-12T23:00:22.000+0000 I pbm-agent:
Version:   2.12.0
Platform:  linux/amd64
GitCommit: 2f7100a2a1da2bb7074e15d757693999cac7574e
GitBranch: release-2.12.0
BuildTime: 2025-10-31_09:21_UTC
GoVersion: go1.25.3
2026-01-12T23:00:22.000+0000 I starting PITR routine
2026-01-12T23:00:22.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.non-voting-and-hidden-8747.svc.cluster.local:27017
2026-01-12T23:00:22.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-01-12T23:00:22.000+0000 I listening for the commands
+ cat /tmp/tmp.QQqqhebuJU
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.EwJIUI7Qwa /tmp/tmp.QQqqhebuJU
+ return 0
+ grep restore-backup-physical
+ tail -100
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CypMXVPVaA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1ibaZDWwpe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CypMXVPVaA
++ cat /tmp/tmp.1ibaZDWwpe
++ rm /tmp/tmp.CypMXVPVaA /tmp/tmp.1ibaZDWwpe
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7b94977d54-82fbc
++ mktemp
+ local LAST_OUT=/tmp/tmp.o57zf30Ymy
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gl0tlgwWqg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7b94977d54-82fbc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.o57zf30Ymy
+ cat /tmp/tmp.Gl0tlgwWqg
+ rm /tmp/tmp.o57zf30Ymy /tmp/tmp.Gl0tlgwWqg
+ return 0
2026-01-12T23:26:18.558Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c7e9a16f-0ab9-496a-90fc-3c69648d783d", "pod": "some-name-rs0-0"}
2026-01-12T23:26:18.558Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c7e9a16f-0ab9-496a-90fc-3c69648d783d", "pod": "some-name-rs0-1"}
2026-01-12T23:26:18.558Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c7e9a16f-0ab9-496a-90fc-3c69648d783d", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:18.558Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c7e9a16f-0ab9-496a-90fc-3c69648d783d", "ready": false}
2026-01-12T23:26:23.559Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:23.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "pod": "some-name-rs0-0"}
2026-01-12T23:26:23.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "pod": "some-name-rs0-1"}
2026-01-12T23:26:23.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "pod": "some-name-rs0-2"}
2026-01-12T23:26:23.560Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:23.560Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "ac48f262-69f9-4461-8f67-ca95610639c5", "ready": false}
2026-01-12T23:26:28.561Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:28.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "pod": "some-name-rs0-0"}
2026-01-12T23:26:28.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "pod": "some-name-rs0-1"}
2026-01-12T23:26:28.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "pod": "some-name-rs0-2"}
2026-01-12T23:26:28.562Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:28.562Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "dca4b42d-ba54-4566-a74d-5e43ae277c78", "ready": false}
2026-01-12T23:26:33.563Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:33.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "pod": "some-name-rs0-2"}
2026-01-12T23:26:33.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "pod": "some-name-rs0-0"}
2026-01-12T23:26:33.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "pod": "some-name-rs0-1"}
2026-01-12T23:26:33.564Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:33.564Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d034cad9-2dd6-46e2-8025-39f6ee1753ec", "ready": false}
2026-01-12T23:26:38.565Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:38.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "pod": "some-name-rs0-0"}
2026-01-12T23:26:38.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "pod": "some-name-rs0-1"}
2026-01-12T23:26:38.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "pod": "some-name-rs0-2"}
2026-01-12T23:26:38.565Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:38.565Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "d11b6e81-3ef9-4360-903d-8cb3d3b9935f", "ready": false}
2026-01-12T23:26:43.567Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:43.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "pod": "some-name-rs0-0"}
2026-01-12T23:26:43.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "pod": "some-name-rs0-1"}
2026-01-12T23:26:43.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "pod": "some-name-rs0-2"}
2026-01-12T23:26:43.567Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:43.567Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "8116fb4d-269f-4534-8f42-d7316cb533f0", "ready": false}
2026-01-12T23:26:48.568Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:48.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "pod": "some-name-rs0-0"}
2026-01-12T23:26:48.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "pod": "some-name-rs0-1"}
2026-01-12T23:26:48.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "pod": "some-name-rs0-2"}
2026-01-12T23:26:48.569Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:48.569Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0deebb92-d1b7-4fdd-a9e0-b586ddc32aff", "ready": false}
2026-01-12T23:26:53.571Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:53.571Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "pod": "some-name-rs0-2"}
2026-01-12T23:26:53.571Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "pod": "some-name-rs0-0"}
2026-01-12T23:26:53.571Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "pod": "some-name-rs0-1"}
2026-01-12T23:26:53.571Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:53.571Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "1d56f131-4c67-4917-9be8-73c470a3aa58", "ready": false}
2026-01-12T23:26:58.573Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:26:58.573Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "pod": "some-name-rs0-0"}
2026-01-12T23:26:58.573Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "pod": "some-name-rs0-1"}
2026-01-12T23:26:58.573Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "pod": "some-name-rs0-2"}
2026-01-12T23:26:58.573Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:26:58.573Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "0a6750f7-6343-41d9-8d11-d79a5db443c7", "ready": false}
2026-01-12T23:27:03.574Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:03.575Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "pod": "some-name-rs0-1"}
2026-01-12T23:27:03.575Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "pod": "some-name-rs0-2"}
2026-01-12T23:27:03.575Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "pod": "some-name-rs0-0"}
2026-01-12T23:27:03.575Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:03.575Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "55a7966a-01f5-4ae3-bd63-bba560b85de2", "ready": false}
2026-01-12T23:27:08.576Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:08.577Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "pod": "some-name-rs0-2"}
2026-01-12T23:27:08.577Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "pod": "some-name-rs0-0"}
2026-01-12T23:27:08.577Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "pod": "some-name-rs0-1"}
2026-01-12T23:27:08.577Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:08.577Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "3dd74f2a-c4fa-4cd5-94e4-e95d656a846e", "ready": false}
2026-01-12T23:27:13.578Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:13.579Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "pod": "some-name-rs0-1"}
2026-01-12T23:27:13.579Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "pod": "some-name-rs0-2"}
2026-01-12T23:27:13.579Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "pod": "some-name-rs0-0"}
2026-01-12T23:27:13.579Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:13.579Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "e2ffc860-22ab-4ed8-8af3-ffb140f7fafa", "ready": false}
2026-01-12T23:27:18.579Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:18.580Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "pod": "some-name-rs0-0"}
2026-01-12T23:27:18.580Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "pod": "some-name-rs0-1"}
2026-01-12T23:27:18.580Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "pod": "some-name-rs0-2"}
2026-01-12T23:27:18.580Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:18.581Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "c8bcd78b-2c19-4b6b-8a86-44fc18f273ef", "ready": false}
2026-01-12T23:27:23.581Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:23.582Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "pod": "some-name-rs0-0"}
2026-01-12T23:27:23.582Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "pod": "some-name-rs0-1"}
2026-01-12T23:27:23.582Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "pod": "some-name-rs0-2"}
2026-01-12T23:27:23.582Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:23.582Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "fe72537f-570d-4141-84ea-6caa45330f45", "ready": false}
2026-01-12T23:27:28.583Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:28.584Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "pod": "some-name-rs0-1"}
2026-01-12T23:27:28.584Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "pod": "some-name-rs0-2"}
2026-01-12T23:27:28.584Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "pod": "some-name-rs0-0"}
2026-01-12T23:27:28.584Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:28.584Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "2ec7f53b-988e-4c67-aafb-7a2a503003c7", "ready": false}
2026-01-12T23:27:33.585Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "pod": "some-name-rs0-1"}
2026-01-12T23:27:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "pod": "some-name-rs0-2"}
2026-01-12T23:27:33.585Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "pod": "some-name-rs0-0"}
2026-01-12T23:27:33.585Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:33.585Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "4e4c25f9-f0a4-4305-b175-60a5953de6e8", "ready": false}
2026-01-12T23:27:38.586Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "cluster": "some-name", "namespace": "non-voting-and-hidden-8747"}
2026-01-12T23:27:38.591Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "pod": "some-name-rs0-0"}
2026-01-12T23:27:38.591Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "pod": "some-name-rs0-1"}
2026-01-12T23:27:38.591Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "pod": "some-name-rs0-2"}
2026-01-12T23:27:38.591Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "sts": "some-name-rs0", "replset": "rs0"}
2026-01-12T23:27:38.591Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"}, "namespace": "non-voting-and-hidden-8747", "name": "restore-backup-physical", "reconcileID": "71120b05-8467-46dd-b8ca-2536fe6900b9", "ready": false}
+ kubectl_bin get psmdb-restore restore-backup-physical -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1EaJLsNfS9
++ mktemp
+ local LAST_ERR=/tmp/tmp.k5gw2t2KSK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-restore restore-backup-physical -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1EaJLsNfS9
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-physical","namespace":"non-voting-and-hidden-8747"},"spec":{"backupName":"backup-physical","clusterName":"some-name"}}
  creationTimestamp: "2026-01-12T22:58:01Z"
  generation: 1
  name: restore-backup-physical
  namespace: non-voting-and-hidden-8747
  resourceVersion: "1768258682367551016"
  uid: 878cef75-b263-4548-8440-6c285c527f30
spec:
  backupName: backup-physical
  clusterName: some-name
status:
  state: waiting
+ cat /tmp/tmp.k5gw2t2KSK
+ rm /tmp/tmp.1EaJLsNfS9 /tmp/tmp.k5gw2t2KSK
+ return 0
+ kubectl_bin describe psmdb-restore restore-backup-physical
++ mktemp
+ local LAST_OUT=/tmp/tmp.fDcmpdFckc
++ mktemp
+ local LAST_ERR=/tmp/tmp.RteC2NipLS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl describe psmdb-restore restore-backup-physical
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fDcmpdFckc
Name:         restore-backup-physical
Namespace:    non-voting-and-hidden-8747
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2026-01-12T22:58:01Z
  Generation:          1
  Resource Version:    1768258682367551016
  UID:                 878cef75-b263-4548-8440-6c285c527f30
Spec:
  Backup Name:   backup-physical
  Cluster Name:  some-name
Status:
  State:  waiting
Events:   <none>
+ cat /tmp/tmp.RteC2NipLS
+ rm /tmp/tmp.fDcmpdFckc /tmp/tmp.RteC2NipLS
+ return 0
+ echo 'Restore object restore-backup-physical is in waiting state after 29 minutes.'
Restore object restore-backup-physical is in waiting state after 29 minutes.
+ echo something went wrong with operator or kubernetes cluster
something went wrong with operator or kubernetes cluster
+ exit 1