Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/logs/non-voting-and-hidden.log
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra non-voting-and-hidden-5889
+ local ns=non-voting-and-hidden-5889
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.yn6Sub86wD
++ mktemp
+ local LAST_ERR=/tmp/tmp.X2GFIaBGRe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yn6Sub86wD
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.X2GFIaBGRe
+ rm /tmp/tmp.yn6Sub86wD /tmp/tmp.X2GFIaBGRe
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.boCKtq4KGr
++ mktemp
+ local LAST_ERR=/tmp/tmp.LyVoBE6Gjd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.boCKtq4KGr
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.LyVoBE6Gjd
+ rm /tmp/tmp.boCKtq4KGr /tmp/tmp.LyVoBE6Gjd
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.XrRddRmzdD
++ mktemp
+ local LAST_ERR=/tmp/tmp.waShnrHK9n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XrRddRmzdD
+ cat /tmp/tmp.waShnrHK9n
+ rm /tmp/tmp.XrRddRmzdD /tmp/tmp.waShnrHK9n
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.G8nfDrX5WP
++ mktemp
+ local LAST_ERR=/tmp/tmp.fKSoRUl9Ac
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G8nfDrX5WP
+ cat /tmp/tmp.fKSoRUl9Ac
+ rm /tmp/tmp.G8nfDrX5WP /tmp/tmp.fKSoRUl9Ac
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.TlFOeAtjIw
++ mktemp
+ local LAST_ERR=/tmp/tmp.11hxKm0DRF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TlFOeAtjIw
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.11hxKm0DRF
+ rm /tmp/tmp.TlFOeAtjIw /tmp/tmp.11hxKm0DRF
+ return 0
+ check_crd_for_deletion PR-2318-8c4144eb
+ local git_tag=PR-2318-8c4144eb
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2318-8c4144eb/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/sbin/sed s/---//g
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bS8ePse0pw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9JOkuFYS8e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bS8ePse0pw
++ cat /tmp/tmp.9JOkuFYS8e
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bS8ePse0pw
++ cat /tmp/tmp.9JOkuFYS8e
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bS8ePse0pw
++ cat /tmp/tmp.9JOkuFYS8e
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.bS8ePse0pw
++ cat /tmp/tmp.9JOkuFYS8e
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.bS8ePse0pw /tmp/tmp.9JOkuFYS8e
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ awk '{print$1}'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.jS87R57YWj
++ mktemp
+ local LAST_OUT=/tmp/tmp.zKdlbmBPf6
+ local LAST_ERR=/tmp/tmp.qZ0YwYju6a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ mktemp
+ local LAST_ERR=/tmp/tmp.KbuoTnzezG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jS87R57YWj
+ cat /tmp/tmp.qZ0YwYju6a
+ rm /tmp/tmp.jS87R57YWj /tmp/tmp.qZ0YwYju6a
+ return 0
namespace "cert-manager" deleted
namespace "non-voting-and-hidden-25782" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zKdlbmBPf6
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.KbuoTnzezG
+ rm /tmp/tmp.zKdlbmBPf6 /tmp/tmp.KbuoTnzezG
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wBnBDNcyDB
++ mktemp
+ local LAST_ERR=/tmp/tmp.AiGrbCrHFo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wBnBDNcyDB
+ cat /tmp/tmp.AiGrbCrHFo
+ rm /tmp/tmp.wBnBDNcyDB /tmp/tmp.AiGrbCrHFo
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9uRhkmYjwE
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wzcz6396a7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9uRhkmYjwE
namespace/psmdb-operator created
+ cat /tmp/tmp.Wzcz6396a7
+ rm /tmp/tmp.9uRhkmYjwE /tmp/tmp.Wzcz6396a7
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H8zCLMtRGJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uZmiegzG0H
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H8zCLMtRGJ
++ cat /tmp/tmp.uZmiegzG0H
++ rm /tmp/tmp.H8zCLMtRGJ /tmp/tmp.uZmiegzG0H
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.JcCr085Z6W
++ mktemp
+ local LAST_ERR=/tmp/tmp.OQDRq7AyOb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JcCr085Z6W
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13" modified.
+ cat /tmp/tmp.OQDRq7AyOb
+ rm /tmp/tmp.JcCr085Z6W /tmp/tmp.OQDRq7AyOb
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2318-8c4144eb'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2318-8c4144eb
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BCsI5nhfcf
++ mktemp
+ local LAST_ERR=/tmp/tmp.a7kzSuCtNU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BCsI5nhfcf
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.a7kzSuCtNU
+ rm /tmp/tmp.BCsI5nhfcf /tmp/tmp.a7kzSuCtNU
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.0kBMrYuN6B
++ mktemp
+ local LAST_ERR=/tmp/tmp.7uLin8wMRe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0kBMrYuN6B
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.7uLin8wMRe
+ rm /tmp/tmp.0kBMrYuN6B /tmp/tmp.7uLin8wMRe
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2318-8c4144eb") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.p662xRvHV6
++ mktemp
+ local LAST_ERR=/tmp/tmp.oSeMyi6jIt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p662xRvHV6
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.oSeMyi6jIt
+ rm /tmp/tmp.p662xRvHV6 /tmp/tmp.oSeMyi6jIt
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qBckHI3EWF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.skHSuwYcUu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qBckHI3EWF
++ cat /tmp/tmp.skHSuwYcUu
++ rm /tmp/tmp.qBckHI3EWF /tmp/tmp.skHSuwYcUu
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-7cb78d7bc6-jfrkg
+ local pod=percona-server-mongodb-operator-7cb78d7bc6-jfrkg
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-7cb78d7bc6-jfrkg to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lr6M8wMLN0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8osuhBpp4K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lr6M8wMLN0
++ cat /tmp/tmp.8osuhBpp4K
++ rm /tmp/tmp.Lr6M8wMLN0 /tmp/tmp.8osuhBpp4K
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7cb78d7bc6-jfrkg
++ mktemp
+ local LAST_OUT=/tmp/tmp.smHXyktZLm
++ mktemp
+ local LAST_ERR=/tmp/tmp.yp5y6iYBm8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7cb78d7bc6-jfrkg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.smHXyktZLm
+ cat /tmp/tmp.yp5y6iYBm8
+ rm /tmp/tmp.smHXyktZLm /tmp/tmp.yp5y6iYBm8
+ return 0
2026-04-24T05:54:50.105Z	INFO	setup	Manager starting up	{"gitCommit": "8c4144ebb25eccddc88123f0a358ed2b0469b4ab", "gitBranch": "PR-2318-8c4144eb", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace non-voting-and-hidden-5889
+ local namespace=non-voting-and-hidden-5889
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces non-voting-and-hidden-5889'
+ xargs kubectl delete ns
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces non-voting-and-hidden-5889
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace non-voting-and-hidden-5889 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.F23tsm9KO1
++ mktemp
+ local LAST_OUT=/tmp/tmp.U1J6w5ka6m
++ mktemp
+ local LAST_ERR=/tmp/tmp.nOt4fpGhKV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Hj93vfNbN3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace non-voting-and-hidden-5889 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F23tsm9KO1
+ cat /tmp/tmp.nOt4fpGhKV
+ rm /tmp/tmp.F23tsm9KO1 /tmp/tmp.nOt4fpGhKV
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U1J6w5ka6m
+ cat /tmp/tmp.Hj93vfNbN3
+ rm /tmp/tmp.U1J6w5ka6m /tmp/tmp.Hj93vfNbN3
+ return 0
+ kubectl_bin wait --for=delete namespace non-voting-and-hidden-5889
++ mktemp
+ local LAST_OUT=/tmp/tmp.r6dZLkTKzb
++ mktemp
+ local LAST_ERR=/tmp/tmp.KNdPMsXCVB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace non-voting-and-hidden-5889
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r6dZLkTKzb
+ cat /tmp/tmp.KNdPMsXCVB
+ rm /tmp/tmp.r6dZLkTKzb /tmp/tmp.KNdPMsXCVB
+ return 0
+ desc 'create namespace non-voting-and-hidden-5889'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace non-voting-and-hidden-5889
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace non-voting-and-hidden-5889
++ mktemp
+ local LAST_OUT=/tmp/tmp.ROFlspl9ut
++ mktemp
+ local LAST_ERR=/tmp/tmp.R3LfzSCSqH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace non-voting-and-hidden-5889
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ROFlspl9ut
namespace/non-voting-and-hidden-5889 created
+ cat /tmp/tmp.R3LfzSCSqH
+ rm /tmp/tmp.ROFlspl9ut /tmp/tmp.R3LfzSCSqH
+ return 0
+ set_kube_ctx non-voting-and-hidden-5889
+ local namespace=non-voting-and-hidden-5889
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lke2ULNuBQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OJlu9eYTFk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lke2ULNuBQ
++ cat /tmp/tmp.OJlu9eYTFk
++ rm /tmp/tmp.Lke2ULNuBQ /tmp/tmp.OJlu9eYTFk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13 --namespace=non-voting-and-hidden-5889
++ mktemp
+ local LAST_OUT=/tmp/tmp.Bs7zLMGU6n
++ mktemp
+ local LAST_ERR=/tmp/tmp.iN7E661g0C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13 --namespace=non-voting-and-hidden-5889
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Bs7zLMGU6n
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2318-8c4144eb-1-cluster13" modified.
+ cat /tmp/tmp.iN7E661g0C
+ rm /tmp/tmp.Bs7zLMGU6n /tmp/tmp.iN7E661g0C
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.8rOuuj3xnY
++ mktemp
+ local LAST_ERR=/tmp/tmp.tMGI5GF6jb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8rOuuj3xnY
namespace/cert-manager created
+ cat /tmp/tmp.tMGI5GF6jb
+ rm /tmp/tmp.8rOuuj3xnY /tmp/tmp.tMGI5GF6jb
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.68W6u2vKxm
++ mktemp
+ local LAST_ERR=/tmp/tmp.aLlA7HaDzW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.68W6u2vKxm
namespace/cert-manager labeled
+ cat /tmp/tmp.aLlA7HaDzW
+ rm /tmp/tmp.68W6u2vKxm /tmp/tmp.aLlA7HaDzW
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.WvSNIWjMCy
++ mktemp
+ local LAST_ERR=/tmp/tmp.pU8IRuixlT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WvSNIWjMCy
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.pU8IRuixlT
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.WvSNIWjMCy /tmp/tmp.pU8IRuixlT
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.lU4OYphdZA
++ mktemp
+ local LAST_ERR=/tmp/tmp.BKjFXoT55Y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lU4OYphdZA
pod/cert-manager-559d798845-qb6nz condition met
pod/cert-manager-cainjector-64958d9c7c-srfvh condition met
pod/cert-manager-webhook-7fb6f99b56-kwdwq condition met
+ cat /tmp/tmp.BKjFXoT55Y
+ rm /tmp/tmp.lU4OYphdZA /tmp/tmp.BKjFXoT55Y
+ return 0
+ sleep 120
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OPuj2goSjE
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Hl0gNJfZG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OPuj2goSjE
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.2Hl0gNJfZG
+ rm /tmp/tmp.OPuj2goSjE /tmp/tmp.2Hl0gNJfZG
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Fri Apr 24 05:57:52 2026
NAMESPACE: non-voting-and-hidden-5889
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.non-voting-and-hidden-5889.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace non-voting-and-hidden-5889 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace non-voting-and-hidden-5889

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace non-voting-and-hidden-5889 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace non-voting-and-hidden-5889 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6krNGFTca8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qj3xRLXvE6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6krNGFTca8
++ cat /tmp/tmp.Qj3xRLXvE6
++ rm /tmp/tmp.6krNGFTca8 /tmp/tmp.Qj3xRLXvE6
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-p89vf
+ wait_pod minio-service-6d5f646cdc-p89vf
+ local pod=minio-service-6d5f646cdc-p89vf
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-p89vf to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.non-voting-and-hidden-5889.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.BZWou5jvTB
++ mktemp
+ local LAST_ERR=/tmp/tmp.MIt2lIQyTJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BZWou5jvTB
make_bucket: operator-testing
pod "aws-cli" deleted from non-voting-and-hidden-5889 namespace
+ cat /tmp/tmp.MIt2lIQyTJ
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.BZWou5jvTB /tmp/tmp.MIt2lIQyTJ
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hUXrCJljFa
++ mktemp
+ local LAST_ERR=/tmp/tmp.zl5p3z7q1j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hUXrCJljFa
deployment.apps/psmdb-client created
secret/some-users created
+ cat /tmp/tmp.zl5p3z7q1j
+ rm /tmp/tmp.hUXrCJljFa /tmp/tmp.zl5p3z7q1j
+ return 0
+ desc 'Creating PSMDB cluster with non-voting + hidden members'
+ set +o xtrace


-----------------------------------------------------------------------------------
Creating PSMDB cluster with non-voting + hidden members
-----------------------------------------------------------------------------------

+ log 'create PSMDB cluster some-name'
+ set +o xtrace
[2026-04-24T05:58:33+0000] create PSMDB cluster some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2318-8c4144eb"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/non-voting-and-hidden-5889/g
+ local LAST_OUT=/tmp/tmp.0RswH38423
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.aBMyO4uzaQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0RswH38423
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.aBMyO4uzaQ
+ rm /tmp/tmp.0RswH38423 /tmp/tmp.aBMyO4uzaQ
+ return 0
+ log 'check if Pods are started'
+ set +o xtrace
[2026-04-24T05:58:35+0000] check if Pods are started
+ wait_for_running some-name-rs0 3 false
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.....OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fV1WeKvKK6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pjcooxIxtD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fV1WeKvKK6
++ cat /tmp/tmp.pjcooxIxtD
++ rm /tmp/tmp.fV1WeKvKK6 /tmp/tmp.pjcooxIxtD
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZvR0kGN7rx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cCk7eOA5uc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZvR0kGN7rx
++ cat /tmp/tmp.cCk7eOA5uc
++ rm /tmp/tmp.ZvR0kGN7rx /tmp/tmp.cCk7eOA5uc
++ return 0
+ [[ true == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.11JdImR5vp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BnVsE4KqNJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.11JdImR5vp
++ cat /tmp/tmp.BnVsE4KqNJ
++ rm /tmp/tmp.11JdImR5vp /tmp/tmp.BnVsE4KqNJ
++ return 0
+ last_pod=0
++ seq 0 0
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-nv-0
+ local pod=some-name-rs0-nv-0
+ set +o xtrace
waiting for pod/some-name-rs0-nv-0 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6VlJwdGhFr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FQJ0IsjcqH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6VlJwdGhFr
++ cat /tmp/tmp.FQJ0IsjcqH
++ rm /tmp/tmp.6VlJwdGhFr /tmp/tmp.FQJ0IsjcqH
++ return 0
+ [[ true == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.size}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GT6On8mrFH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hY2PSeiAmK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.size}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GT6On8mrFH
++ cat /tmp/tmp.hY2PSeiAmK
++ rm /tmp/tmp.GT6On8mrFH /tmp/tmp.hY2PSeiAmK
++ return 0
+ last_pod=1
++ seq 0 1
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-hidden-0
+ local pod=some-name-rs0-hidden-0
+ set +o xtrace
waiting for pod/some-name-rs0-hidden-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ wait_pod some-name-rs0-hidden-1
+ local pod=some-name-rs0-hidden-1
+ set +o xtrace
waiting for pod/some-name-rs0-hidden-1 to be ready.OK
+ sleep 10
+ [[ false == true ]]
+ wait_for_cluster_state some-name ready
+ local cluster_name=some-name
+ local target_state=ready
+ echo -n 'Waiting for psmdb/some-name to reach ready state'
Waiting for psmdb/some-name to reach ready state+ local timeout=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6qAFv4iZlM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1c4C8e0QS9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6qAFv4iZlM
++ cat /tmp/tmp.1c4C8e0QS9
++ rm /tmp/tmp.6qAFv4iZlM /tmp/tmp.1c4C8e0QS9
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=1
+ echo -n .
.+ [[ 1 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D3RZr8W3XX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hQa2hBmJUe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D3RZr8W3XX
++ cat /tmp/tmp.hQa2hBmJUe
++ rm /tmp/tmp.D3RZr8W3XX /tmp/tmp.hQa2hBmJUe
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=2
+ echo -n .
.+ [[ 2 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GdxIbv2UiX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kw1VWSKKCX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GdxIbv2UiX
++ cat /tmp/tmp.Kw1VWSKKCX
++ rm /tmp/tmp.GdxIbv2UiX /tmp/tmp.Kw1VWSKKCX
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=3
+ echo -n .
.+ [[ 3 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6qrWKqOQHY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.urJYPrAoCF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6qrWKqOQHY
++ cat /tmp/tmp.urJYPrAoCF
++ rm /tmp/tmp.6qrWKqOQHY /tmp/tmp.urJYPrAoCF
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=4
+ echo -n .
.+ [[ 4 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HROGcVXCVR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mIisMjCEy4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HROGcVXCVR
++ cat /tmp/tmp.mIisMjCEy4
++ rm /tmp/tmp.HROGcVXCVR /tmp/tmp.mIisMjCEy4
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=5
+ echo -n .
.+ [[ 5 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mSOJnJyrX4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ydEVTDx86O
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mSOJnJyrX4
++ cat /tmp/tmp.ydEVTDx86O
++ rm /tmp/tmp.mSOJnJyrX4 /tmp/tmp.ydEVTDx86O
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=6
+ echo -n .
.+ [[ 6 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V41hawdzrC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2Y2wajRzAz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V41hawdzrC
++ cat /tmp/tmp.2Y2wajRzAz
++ rm /tmp/tmp.V41hawdzrC /tmp/tmp.2Y2wajRzAz
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=7
+ echo -n .
.+ [[ 7 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.57J9zPAVVU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ep6k1Ikxzt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.57J9zPAVVU
++ cat /tmp/tmp.ep6k1Ikxzt
++ rm /tmp/tmp.57J9zPAVVU /tmp/tmp.ep6k1Ikxzt
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=8
+ echo -n .
.+ [[ 8 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I8JteGVTgf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ccrn66QPXK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I8JteGVTgf
++ cat /tmp/tmp.Ccrn66QPXK
++ rm /tmp/tmp.I8JteGVTgf /tmp/tmp.Ccrn66QPXK
++ return 0
+ [[ error =~ ready ]]
+ sleep 1
+ timeout=9
+ echo -n .
.+ [[ 9 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5J9pswWgqw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YUVEBq0oTo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5J9pswWgqw
++ cat /tmp/tmp.YUVEBq0oTo
++ rm /tmp/tmp.5J9pswWgqw /tmp/tmp.YUVEBq0oTo
++ return 0
+ [[ error =~ ready ]]
+ sleep 1
+ timeout=10
+ echo -n .
.+ [[ 10 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zeYNgtiSUf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YPURbBHp39
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zeYNgtiSUf
++ cat /tmp/tmp.YPURbBHp39
++ rm /tmp/tmp.zeYNgtiSUf /tmp/tmp.YPURbBHp39
++ return 0
+ [[ error =~ ready ]]
+ sleep 1
+ timeout=11
+ echo -n .
.+ [[ 11 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vpTSQZLK8G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.61uWE2CgtI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vpTSQZLK8G
++ cat /tmp/tmp.61uWE2CgtI
++ rm /tmp/tmp.vpTSQZLK8G /tmp/tmp.61uWE2CgtI
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=12
+ echo -n .
.+ [[ 12 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L2PdrUl8US
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pqbdjZ6Ycm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L2PdrUl8US
++ cat /tmp/tmp.pqbdjZ6Ycm
++ rm /tmp/tmp.L2PdrUl8US /tmp/tmp.pqbdjZ6Ycm
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=13
+ echo -n .
.+ [[ 13 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ErJ4mTYfRP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tGOYcw3uUK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ErJ4mTYfRP
++ cat /tmp/tmp.tGOYcw3uUK
++ rm /tmp/tmp.ErJ4mTYfRP /tmp/tmp.tGOYcw3uUK
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=14
+ echo -n .
.+ [[ 14 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4cnioCOAZt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0GJTzF8ATc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4cnioCOAZt
++ cat /tmp/tmp.0GJTzF8ATc
++ rm /tmp/tmp.4cnioCOAZt /tmp/tmp.0GJTzF8ATc
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=15
+ echo -n .
.+ [[ 15 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.szeJi8moIr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sBHYZOMBfy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.szeJi8moIr
++ cat /tmp/tmp.sBHYZOMBfy
++ rm /tmp/tmp.szeJi8moIr /tmp/tmp.sBHYZOMBfy
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=16
+ echo -n .
.+ [[ 16 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gnhuOGUcLo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F77wDU6ZSC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gnhuOGUcLo
++ cat /tmp/tmp.F77wDU6ZSC
++ rm /tmp/tmp.gnhuOGUcLo /tmp/tmp.F77wDU6ZSC
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=17
+ echo -n .
.+ [[ 17 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tc0WBFY0ii
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8WKuHduKnX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tc0WBFY0ii
++ cat /tmp/tmp.8WKuHduKnX
++ rm /tmp/tmp.Tc0WBFY0ii /tmp/tmp.8WKuHduKnX
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=18
+ echo -n .
.+ [[ 18 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OTla2Ung0Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x2y6MA4CQT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OTla2Ung0Y
++ cat /tmp/tmp.x2y6MA4CQT
++ rm /tmp/tmp.OTla2Ung0Y /tmp/tmp.x2y6MA4CQT
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=19
+ echo -n .
.+ [[ 19 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XOM7Z2j3RT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QBETYJ3ipo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XOM7Z2j3RT
++ cat /tmp/tmp.QBETYJ3ipo
++ rm /tmp/tmp.XOM7Z2j3RT /tmp/tmp.QBETYJ3ipo
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=20
+ echo -n .
.+ [[ 20 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.31bHYXA63u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s2YyRf76YU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.31bHYXA63u
++ cat /tmp/tmp.s2YyRf76YU
++ rm /tmp/tmp.31bHYXA63u /tmp/tmp.s2YyRf76YU
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=21
+ echo -n .
.+ [[ 21 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kbni9ANva2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VOGjvrrmWk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kbni9ANva2
++ cat /tmp/tmp.VOGjvrrmWk
++ rm /tmp/tmp.Kbni9ANva2 /tmp/tmp.VOGjvrrmWk
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=22
+ echo -n .
.+ [[ 22 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3maKgFTQrW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N6OFDR3jeQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3maKgFTQrW
++ cat /tmp/tmp.N6OFDR3jeQ
++ rm /tmp/tmp.3maKgFTQrW /tmp/tmp.N6OFDR3jeQ
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=23
+ echo -n .
.+ [[ 23 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vXQiVeX6nv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.81OG7Hkpe2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vXQiVeX6nv
++ cat /tmp/tmp.81OG7Hkpe2
++ rm /tmp/tmp.vXQiVeX6nv /tmp/tmp.81OG7Hkpe2
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=24
+ echo -n .
.+ [[ 24 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JceIDMsFlA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oQqWwZAlfP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JceIDMsFlA
++ cat /tmp/tmp.oQqWwZAlfP
++ rm /tmp/tmp.JceIDMsFlA /tmp/tmp.oQqWwZAlfP
++ return 0
+ [[ initializing =~ ready ]]
+ sleep 1
+ timeout=25
+ echo -n .
.+ [[ 25 -gt 1500 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UCBtfvS45d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EShZm4GXOd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UCBtfvS45d
++ cat /tmp/tmp.EShZm4GXOd
++ rm /tmp/tmp.UCBtfvS45d /tmp/tmp.EShZm4GXOd
++ return 0
+ [[ ready =~ ready ]]
+ echo

+ log 'psmdb/some-name is ready: OK'
+ set +o xtrace
[2026-04-24T06:00:51+0000] psmdb/some-name is ready: OK
+ log 'check if statefulsets created with expected config'
+ set +o xtrace
[2026-04-24T06:00:51+0000] check if statefulsets created with expected config
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-5889", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wT01EdXySf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kzc90aeVHv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wT01EdXySf
+ cat /tmp/tmp.Kzc90aeVHv
+ rm /tmp/tmp.wT01EdXySf /tmp/tmp.Kzc90aeVHv
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0.yml /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-24T06:00:52+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-rs0-nv
+ local resource=statefulset/some-name-rs0-nv
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml
+ local new_result=/tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-nv.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0-nv
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-5889", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Di5jkEGoV0
++ mktemp
+ local LAST_ERR=/tmp/tmp.qQ7H8asQR2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0-nv
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Di5jkEGoV0
+ cat /tmp/tmp.qQ7H8asQR2
+ rm /tmp/tmp.Di5jkEGoV0 /tmp/tmp.qQ7H8asQR2
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-nv.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-nv.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-nv.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-nv.yml /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-nv.yml
+ log 'compare_kubectl: statefulset/some-name-rs0-nv OK'
+ set +o xtrace
[2026-04-24T06:00:53+0000] compare_kubectl: statefulset/some-name-rs0-nv OK
+ compare_kubectl statefulset/some-name-rs0-hidden
+ local resource=statefulset/some-name-rs0-hidden
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml
+ local new_result=/tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-hidden.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0-hidden
++ mktemp
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("non-voting-and-hidden-5889", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.Iz6bNdRAzU
++ mktemp
+ local LAST_ERR=/tmp/tmp.1sjsDdyGI5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0-hidden
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Iz6bNdRAzU
+ cat /tmp/tmp.1sjsDdyGI5
+ rm /tmp/tmp.Iz6bNdRAzU /tmp/tmp.1sjsDdyGI5
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-hidden.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-hidden.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-hidden.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/statefulset_some-name-rs0-hidden.yml /tmp/tmp.IbsKxb3OGY/statefulset_some-name-rs0-hidden.yml
+ log 'compare_kubectl: statefulset/some-name-rs0-hidden OK'
+ set +o xtrace
[2026-04-24T06:00:55+0000] compare_kubectl: statefulset/some-name-rs0-hidden OK
+ log 'check if rs.config is correct'
+ set +o xtrace
[2026-04-24T06:00:55+0000] check if rs.config is correct
+ compare_rs_config 'rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 nonvoting
+ local 'command=rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local compare_file=nonvoting
+ run_mongo 'rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 mongodb ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1W5QHgUCB5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2czjSx2anl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1W5QHgUCB5
++ cat /tmp/tmp.2czjSx2anl
++ rm /tmp/tmp.1W5QHgUCB5 /tmp/tmp.2czjSx2anl
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eETKp4CT8i
++ mktemp
+ local LAST_ERR=/tmp/tmp.QtOTpqRuvc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''rs.config().members.filter(m => m.tags.nonVoting == "true").map(m => m.votes)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eETKp4CT8i
+ cat /tmp/tmp.QtOTpqRuvc
+ rm /tmp/tmp.eETKp4CT8i /tmp/tmp.QtOTpqRuvc
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/nonvoting.json /tmp/tmp.IbsKxb3OGY/nonvoting
+ log 'nonVoting members: OK'
+ set +o xtrace
[2026-04-24T06:00:57+0000] nonVoting members: OK
+ compare_rs_config 'rs.config().members.map(m => m.hidden)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 hidden
+ local 'command=rs.config().members.map(m => m.hidden)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local compare_file=hidden
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'rs.config().members.map(m => m.hidden)' clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 mongodb ''
+ local 'command=rs.config().members.map(m => m.hidden)'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MXuRcJDqk3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NfFwODONLr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MXuRcJDqk3
++ cat /tmp/tmp.NfFwODONLr
++ rm /tmp/tmp.MXuRcJDqk3 /tmp/tmp.NfFwODONLr
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''rs.config().members.map(m => m.hidden)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IXlRl2WLco
++ mktemp
+ local LAST_ERR=/tmp/tmp.0q2pkJuThx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''rs.config().members.map(m => m.hidden)\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IXlRl2WLco
+ cat /tmp/tmp.0q2pkJuThx
+ rm /tmp/tmp.IXlRl2WLco /tmp/tmp.0q2pkJuThx
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/hidden.json /tmp/tmp.IbsKxb3OGY/hidden
+ log 'hidden members: OK'
+ set +o xtrace
[2026-04-24T06:01:00+0000] hidden members: OK
+ log 'create user myApp'
+ set +o xtrace
[2026-04-24T06:01:00+0000] create user myApp
+ run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local 'command=db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aTpJPbajnM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wJiCBs64h5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aTpJPbajnM
++ cat /tmp/tmp.wJiCBs64h5
++ rm /tmp/tmp.aTpJPbajnM /tmp/tmp.wJiCBs64h5
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4uL0EB3a0f
++ mktemp
+ local LAST_ERR=/tmp/tmp.BfG9hhXHsE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4uL0EB3a0f
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("6a3fae00-2479-40d1-b9cf-775704991ecd") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.BfG9hhXHsE
+ rm /tmp/tmp.4uL0EB3a0f /tmp/tmp.BfG9hhXHsE
+ return 0
+ log 'write data'
+ set +o xtrace
[2026-04-24T06:01:02+0000] write data
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OYqaVoN5u2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9nhz8WWbcv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OYqaVoN5u2
++ cat /tmp/tmp.9nhz8WWbcv
++ rm /tmp/tmp.OYqaVoN5u2 /tmp/tmp.9nhz8WWbcv
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3lu3ocvSNi
++ mktemp
+ local LAST_ERR=/tmp/tmp.0g3rIFKDV0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3lu3ocvSNi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("27367a03-07f8-46ca-8bb0-679a84e31012") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.0g3rIFKDV0
+ rm /tmp/tmp.3lu3ocvSNi /tmp/tmp.0g3rIFKDV0
+ return 0
+ sleep 10
+ log 'compare data'
+ set +o xtrace
[2026-04-24T06:01:15+0000] compare data
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:01:15+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hn8AnRDv84
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h0EskkMAY9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hn8AnRDv84
++ cat /tmp/tmp.h0EskkMAY9
++ rm /tmp/tmp.hn8AnRDv84 /tmp/tmp.h0EskkMAY9
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PltRUubmyb
++ mktemp
+ local LAST_ERR=/tmp/tmp.cZYgule3fT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PltRUubmyb
+ cat /tmp/tmp.cZYgule3fT
+ rm /tmp/tmp.PltRUubmyb /tmp/tmp.cZYgule3fT
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:01:18+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7RpJ6GJEYy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bUnlNUZc9h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7RpJ6GJEYy
++ cat /tmp/tmp.bUnlNUZc9h
++ rm /tmp/tmp.7RpJ6GJEYy /tmp/tmp.bUnlNUZc9h
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZEAVaOmoav
++ mktemp
+ local LAST_ERR=/tmp/tmp.QYotygMgUw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZEAVaOmoav
+ cat /tmp/tmp.QYotygMgUw
+ rm /tmp/tmp.ZEAVaOmoav /tmp/tmp.QYotygMgUw
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:01:20+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vbJN4B2jZ9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J4AkqgPTco
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vbJN4B2jZ9
++ cat /tmp/tmp.J4AkqgPTco
++ rm /tmp/tmp.vbJN4B2jZ9 /tmp/tmp.J4AkqgPTco
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RhFKsXcZDn
++ mktemp
+ local LAST_ERR=/tmp/tmp.JtKqFqZN2m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RhFKsXcZDn
+ cat /tmp/tmp.JtKqFqZN2m
+ rm /tmp/tmp.RhFKsXcZDn /tmp/tmp.JtKqFqZN2m
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:01:23+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8BQVwEMKBQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wrw9Ne6HCN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8BQVwEMKBQ
++ cat /tmp/tmp.Wrw9Ne6HCN
++ rm /tmp/tmp.8BQVwEMKBQ /tmp/tmp.Wrw9Ne6HCN
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Fi9ZNa4onb
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZTi7Vs2w7B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Fi9ZNa4onb
+ cat /tmp/tmp.ZTi7Vs2w7B
+ rm /tmp/tmp.Fi9ZNa4onb /tmp/tmp.ZTi7Vs2w7B
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ run_backup minio backup-logical logical
+ local storage=minio
+ local backup_name=backup-logical
+ local type=logical
+ log 'running backup backup-logical'
+ set +o xtrace
[2026-04-24T06:01:25+0000] running backup backup-logical
+ yq eval $'.metadata.name = "backup-logical"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bT6NhnebT3
++ mktemp
+ local LAST_ERR=/tmp/tmp.WF9FfYrsZX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bT6NhnebT3
perconaservermongodbbackup.psmdb.percona.com/backup-logical created
+ cat /tmp/tmp.WF9FfYrsZX
+ rm /tmp/tmp.bT6NhnebT3 /tmp/tmp.WF9FfYrsZX
+ return 0
+ wait_backup backup-logical ready
+ local backup_name=backup-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-logical to reach ready state.......OK
+ log 'drop collection'
+ set +o xtrace
[2026-04-24T06:01:39+0000] drop collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eJrqZM8cRz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ciggS1j7zS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eJrqZM8cRz
++ cat /tmp/tmp.ciggS1j7zS
++ rm /tmp/tmp.eJrqZM8cRz /tmp/tmp.ciggS1j7zS
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bbDtQuOChw
++ mktemp
+ local LAST_ERR=/tmp/tmp.SDElH5tJxl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bbDtQuOChw
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("69fba839-46a8-46ec-bd4c-f0a578a954d9") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.SDElH5tJxl
+ rm /tmp/tmp.bbDtQuOChw /tmp/tmp.SDElH5tJxl
+ return 0
+ run_restore backup-logical
+ local backup_name=backup-logical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ log 'running restore restore-backup-logical'
+ set +o xtrace
[2026-04-24T06:01:42+0000] running restore restore-backup-logical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ kubectl_bin apply -f -
+ /usr/sbin/sed -e 's/name:/name: restore-backup-logical/'
++ mktemp
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-logical/'
+ local LAST_OUT=/tmp/tmp.k06pZVHZlr
++ mktemp
+ local LAST_ERR=/tmp/tmp.6siVIhrEOa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k06pZVHZlr
perconaservermongodbrestore.psmdb.percona.com/restore-backup-logical created
+ cat /tmp/tmp.6siVIhrEOa
+ rm /tmp/tmp.k06pZVHZlr /tmp/tmp.6siVIhrEOa
+ return 0
+ wait_restore backup-logical some-name
+ local backup_name=backup-logical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-logical object to be created.OK
Waiting psmdb-restore/restore-backup-logical to reach state "ready" .OK after 0 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qjh4XXXj6e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bXBYLvLGYp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Qjh4XXXj6e
++ cat /tmp/tmp.bXBYLvLGYp
++ rm /tmp/tmp.Qjh4XXXj6e /tmp/tmp.bXBYLvLGYp
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ log 'compare data after restore-backup-logical'
+ set +o xtrace
[2026-04-24T06:02:04+0000] compare data after restore-backup-logical
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:02:04+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lebeT2utnL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9fAxhqCREr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lebeT2utnL
++ cat /tmp/tmp.9fAxhqCREr
++ rm /tmp/tmp.lebeT2utnL /tmp/tmp.9fAxhqCREr
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.B7i52KVFDZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.XNMpLmw6IQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B7i52KVFDZ
+ cat /tmp/tmp.XNMpLmw6IQ
+ rm /tmp/tmp.B7i52KVFDZ /tmp/tmp.XNMpLmw6IQ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:02:06+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P66sx6wdkI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PVjOrEFi4t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P66sx6wdkI
++ cat /tmp/tmp.PVjOrEFi4t
++ rm /tmp/tmp.P66sx6wdkI /tmp/tmp.PVjOrEFi4t
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.5RkPr4ebN8
++ mktemp
+ local LAST_ERR=/tmp/tmp.it1mEemqAj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5RkPr4ebN8
+ cat /tmp/tmp.it1mEemqAj
+ rm /tmp/tmp.5RkPr4ebN8 /tmp/tmp.it1mEemqAj
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:02:09+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nhjyqQtoTl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h2s2uOHCyH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nhjyqQtoTl
++ cat /tmp/tmp.h2s2uOHCyH
++ rm /tmp/tmp.nhjyqQtoTl /tmp/tmp.h2s2uOHCyH
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GLEZ6FIuTW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xtyrqxd1Nk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GLEZ6FIuTW
+ cat /tmp/tmp.Xtyrqxd1Nk
+ rm /tmp/tmp.GLEZ6FIuTW /tmp/tmp.Xtyrqxd1Nk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local tls=false
+ local replicaset=
+ local 'full_command=db.test.find()'
+ [[ -n '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2026-04-24T06:02:11+0000] running db.test.find() in myApp
+ [[ false == true ]]
+ mongo_command=run_mongo
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889 mongodb '' ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
+ grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RN3jvLB8uU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DzJE060i0U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RN3jvLB8uU
++ cat /tmp/tmp.DzJE060i0U
++ rm /tmp/tmp.RN3jvLB8uU /tmp/tmp.DzJE060i0U
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vDWkq4qD2v
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kopsf3qUvC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vDWkq4qD2v
+ cat /tmp/tmp.Kopsf3qUvC
+ rm /tmp/tmp.vDWkq4qD2v /tmp/tmp.Kopsf3qUvC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/compare/find.json /tmp/tmp.IbsKxb3OGY/find
+ run_backup minio backup-physical physical
+ local storage=minio
+ local backup_name=backup-physical
+ local type=physical
+ log 'running backup backup-physical'
+ set +o xtrace
[2026-04-24T06:02:14+0000] running backup backup-physical
+ yq eval $'.metadata.name = "backup-physical"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KHLDrkVvRH
++ mktemp
+ local LAST_ERR=/tmp/tmp.nIzqfSYa5K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KHLDrkVvRH
perconaservermongodbbackup.psmdb.percona.com/backup-physical created
+ cat /tmp/tmp.nIzqfSYa5K
+ rm /tmp/tmp.KHLDrkVvRH /tmp/tmp.nIzqfSYa5K
+ return 0
+ wait_backup backup-physical ready
+ local backup_name=backup-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-physical to reach ready state.......OK
+ log 'drop collection'
+ set +o xtrace
[2026-04-24T06:02:28+0000] drop collection
+ run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-rs0.non-voting-and-hidden-5889
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.non-voting-and-hidden-5889 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y0RUm1f19f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5DbPENaH8m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y0RUm1f19f
++ cat /tmp/tmp.5DbPENaH8m
++ rm /tmp/tmp.y0RUm1f19f /tmp/tmp.5DbPENaH8m
++ return 0
+ local client_container=psmdb-client-bb8b97679-hmhrd
+ kubectl_bin exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kuWI8tx0mg
++ mktemp
+ local LAST_ERR=/tmp/tmp.uzcI1swTWf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-hmhrd -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kuWI8tx0mg
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-hidden-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-hidden-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-nv-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d0f7fcf4-7f2a-4ab1-bbbe-3a3189c7307f") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.uzcI1swTWf
+ rm /tmp/tmp.kuWI8tx0mg /tmp/tmp.uzcI1swTWf
+ return 0
+ run_restore backup-physical
+ local backup_name=backup-physical
+ local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ log 'running restore restore-backup-physical'
+ set +o xtrace
[2026-04-24T06:02:31+0000] running restore restore-backup-physical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2318/e2e-tests/non-voting-and-hidden/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-physical/'
+ /usr/sbin/sed -e 's/name:/name: restore-backup-physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Cx5q4HWzU6
++ mktemp
+ local LAST_ERR=/tmp/tmp.Oe5SArRHzR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cx5q4HWzU6
perconaservermongodbrestore.psmdb.percona.com/restore-backup-physical created
+ cat /tmp/tmp.Oe5SArRHzR
+ rm /tmp/tmp.Cx5q4HWzU6 /tmp/tmp.Oe5SArRHzR
+ return 0
+ wait_restore backup-physical some-name
+ local backup_name=backup-physical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-physical object to be created.OK
Waiting psmdb-restore/restore-backup-physical to reach state "ready" .............................

-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-0
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.vDLE2CZbIc
++ mktemp
+ local LAST_ERR=/tmp/tmp.6SZkJRZ1Qs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vDLE2CZbIc
2026/04/24 06:06:00 [entrypoint] starting `pbm-agent`
2026-04-24T06:06:01.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb
2026-04-24T06:06:01.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-04-24T06:06:01.000+0000 I pbm-agent:
Version:   2.13.0
Platform:  linux/amd64
GitCommit: e9b57e1395fe2602bd9637d5e4ef19b1df2bb3b7
GitBranch: release-2.13.0
BuildTime: 2026-02-26_12:41_UTC
GoVersion: go1.25.7
2026-04-24T06:06:01.000+0000 I starting PITR routine
2026-04-24T06:06:01.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017
2026-04-24T06:06:01.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-04-24T06:06:01.000+0000 I listening for the commands
+ cat /tmp/tmp.6SZkJRZ1Qs
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.vDLE2CZbIc /tmp/tmp.6SZkJRZ1Qs
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-1
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.5mLXRUKvyJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.4zRNgwZnT6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5mLXRUKvyJ
2026/04/24 06:05:25 [entrypoint] starting `pbm-agent`
2026-04-24T06:05:27.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb
2026-04-24T06:05:27.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-04-24T06:05:27.000+0000 I pbm-agent:
Version:   2.13.0
Platform:  linux/amd64
GitCommit: e9b57e1395fe2602bd9637d5e4ef19b1df2bb3b7
GitBranch: release-2.13.0
BuildTime: 2026-02-26_12:41_UTC
GoVersion: go1.25.7
2026-04-24T06:05:27.000+0000 I starting PITR routine
2026-04-24T06:05:27.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017
2026-04-24T06:05:27.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-04-24T06:05:27.000+0000 I listening for the commands
+ cat /tmp/tmp.4zRNgwZnT6
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.5mLXRUKvyJ /tmp/tmp.4zRNgwZnT6
+ return 0
+ for pod in $(get_mongod_pods ${cluster})
+ desc 'pbm-agent logs from some-name-rs0-2'
+ set +o xtrace


-----------------------------------------------------------------------------------
pbm-agent logs from some-name-rs0-2
-----------------------------------------------------------------------------------

+ kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
++ mktemp
+ local LAST_OUT=/tmp/tmp.mCEpMFSjKE
++ mktemp
+ local LAST_ERR=/tmp/tmp.EkGXsd0Atl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mCEpMFSjKE
2026/04/24 06:04:47 [entrypoint] starting `pbm-agent`
2026-04-24T06:04:49.000+0000 I 
                    %                        _____
                   %%%                      |  __ \
                 ###%%%%%%%%%%%%*           | |__) |__ _ __ ___ ___  _ __   __ _
                ###  ##%%      %%%%         |  ___/ _ \ '__/ __/ _ \| '_ \ / _` |
              ####     ##%       %%%%       | |  |  __/ | | (_| (_) | | | | (_| |
             ###        ####      %%%       |_|   \___|_|  \___\___/|_| |_|\__,_|
           ,((###         ###     %%%         _____                       _
          (((( (###        ####  %%%%        / ____|                     | |
         (((     ((#         ######         | (___   __ _ _   _  __ _  __| |
       ((((       (((#        ####           \___ \ / _` | | | |/ _` |/ _` |
      /((          ,(((        *###          ____) | (_| | |_| | (_| | (_| |
    ////             (((         ####       |_____/ \__, |\__,_|\__,_|\__,_|
   ///                ((((        ####                 | |
 /////////////(((((((((((((((((########                |_|   Join @ squad.percona.com/mongodb

** Join Percona Squad! **
Participate in monthly SWAG raffles, get early access to new product features,
invite-only ”ask me anything” sessions with database performance experts.

Interested? Fill in the form at squad.percona.com/mongodb
2026-04-24T06:04:49.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false
2026-04-24T06:04:49.000+0000 I pbm-agent:
Version:   2.13.0
Platform:  linux/amd64
GitCommit: e9b57e1395fe2602bd9637d5e4ef19b1df2bb3b7
GitBranch: release-2.13.0
BuildTime: 2026-02-26_12:41_UTC
GoVersion: go1.25.7
2026-04-24T06:04:49.000+0000 I starting PITR routine
2026-04-24T06:04:49.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.non-voting-and-hidden-5889.svc.cluster.local:27017
2026-04-24T06:04:49.000+0000 I conn level ReadConcern: majority; WriteConcern: majority
2026-04-24T06:04:49.000+0000 I listening for the commands
+ cat /tmp/tmp.EkGXsd0Atl
Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init)
+ rm /tmp/tmp.mCEpMFSjKE /tmp/tmp.EkGXsd0Atl
+ return 0
+ grep restore-backup-physical
+ tail -100
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kga4W0PUgt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SeWnTrad3M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kga4W0PUgt
++ cat /tmp/tmp.SeWnTrad3M
++ rm /tmp/tmp.Kga4W0PUgt /tmp/tmp.SeWnTrad3M
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-7cb78d7bc6-jfrkg
++ mktemp
+ local LAST_OUT=/tmp/tmp.LavmZQyGir
++ mktemp
+ local LAST_ERR=/tmp/tmp.b5a4KC5Dew
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-7cb78d7bc6-jfrkg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LavmZQyGir
+ cat /tmp/tmp.b5a4KC5Dew
+ rm /tmp/tmp.LavmZQyGir /tmp/tmp.b5a4KC5Dew
+ return 0
2026-04-24T06:30:50.541Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "31ead6b1-6cfe-491f-80fc-175d44935e66", "pod": "some-name-rs0-2"}
2026-04-24T06:30:50.541Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "31ead6b1-6cfe-491f-80fc-175d44935e66", "pod": "some-name-rs0-0"}
2026-04-24T06:30:50.541Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "31ead6b1-6cfe-491f-80fc-175d44935e66", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:30:50.542Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "31ead6b1-6cfe-491f-80fc-175d44935e66", "ready": false}
2026-04-24T06:30:55.543Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:30:55.543Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "pod": "some-name-rs0-1"}
2026-04-24T06:30:55.543Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "pod": "some-name-rs0-2"}
2026-04-24T06:30:55.543Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "pod": "some-name-rs0-0"}
2026-04-24T06:30:55.543Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:30:55.543Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "53bc0bc6-b26c-4142-b50b-e58c04a4e2f1", "ready": false}
2026-04-24T06:31:00.544Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:00.545Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "pod": "some-name-rs0-2"}
2026-04-24T06:31:00.545Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "pod": "some-name-rs0-0"}
2026-04-24T06:31:00.545Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "pod": "some-name-rs0-1"}
2026-04-24T06:31:00.545Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:00.545Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "3e4da883-9a44-4c1c-99c3-625167910ba8", "ready": false}
2026-04-24T06:31:05.546Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:05.546Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "pod": "some-name-rs0-1"}
2026-04-24T06:31:05.546Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "pod": "some-name-rs0-2"}
2026-04-24T06:31:05.546Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "pod": "some-name-rs0-0"}
2026-04-24T06:31:05.546Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:05.546Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "18a5b354-8583-4d3c-8106-bfe95d546a68", "ready": false}
2026-04-24T06:31:10.547Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:10.548Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "pod": "some-name-rs0-0"}
2026-04-24T06:31:10.548Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "pod": "some-name-rs0-1"}
2026-04-24T06:31:10.548Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "pod": "some-name-rs0-2"}
2026-04-24T06:31:10.548Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:10.548Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "84b40f60-4c8a-4240-8380-d43fa6b9f3b8", "ready": false}
2026-04-24T06:31:15.549Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:15.550Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "pod": "some-name-rs0-2"}
2026-04-24T06:31:15.550Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "pod": "some-name-rs0-0"}
2026-04-24T06:31:15.550Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "pod": "some-name-rs0-1"}
2026-04-24T06:31:15.550Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:15.550Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e7e2cfee-bf6a-4af8-92fa-481278f5f6bc", "ready": false}
2026-04-24T06:31:20.550Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:20.551Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "pod": "some-name-rs0-1"}
2026-04-24T06:31:20.551Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "pod": "some-name-rs0-2"}
2026-04-24T06:31:20.551Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "pod": "some-name-rs0-0"}
2026-04-24T06:31:20.551Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:20.551Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "4f1b56e0-4366-4ebe-ae6c-2e5bf7d20d5b", "ready": false}
2026-04-24T06:31:25.552Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:25.553Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "pod": "some-name-rs0-2"}
2026-04-24T06:31:25.553Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "pod": "some-name-rs0-0"}
2026-04-24T06:31:25.553Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "pod": "some-name-rs0-1"}
2026-04-24T06:31:25.553Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:25.553Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "210d492b-dc36-4aac-ac6c-53f6c5f9d372", "ready": false}
2026-04-24T06:31:30.554Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:30.555Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "pod": "some-name-rs0-1"}
2026-04-24T06:31:30.555Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "pod": "some-name-rs0-2"}
2026-04-24T06:31:30.555Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "pod": "some-name-rs0-0"}
2026-04-24T06:31:30.555Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:30.555Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "92aace58-e5f7-4221-9aa1-4b786b4c5619", "ready": false}
2026-04-24T06:31:35.556Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:35.557Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "pod": "some-name-rs0-2"}
2026-04-24T06:31:35.557Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "pod": "some-name-rs0-0"}
2026-04-24T06:31:35.557Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "pod": "some-name-rs0-1"}
2026-04-24T06:31:35.557Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:35.557Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "55c9a6ae-1930-48b1-b887-350b49b14065", "ready": false}
2026-04-24T06:31:40.558Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:40.558Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "pod": "some-name-rs0-2"}
2026-04-24T06:31:40.558Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "pod": "some-name-rs0-0"}
2026-04-24T06:31:40.558Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "pod": "some-name-rs0-1"}
2026-04-24T06:31:40.558Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:40.559Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "04b3e9ce-1bd8-4498-903b-82bd4c7a7e6c", "ready": false}
2026-04-24T06:31:45.559Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:45.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "pod": "some-name-rs0-1"}
2026-04-24T06:31:45.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "pod": "some-name-rs0-2"}
2026-04-24T06:31:45.560Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "pod": "some-name-rs0-0"}
2026-04-24T06:31:45.560Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:45.560Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "7b9e0262-ad11-482f-95df-d0d8376ccc96", "ready": false}
2026-04-24T06:31:50.561Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:50.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "pod": "some-name-rs0-1"}
2026-04-24T06:31:50.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "pod": "some-name-rs0-2"}
2026-04-24T06:31:50.562Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "pod": "some-name-rs0-0"}
2026-04-24T06:31:50.562Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:50.562Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "e880f5f4-8463-48d4-8aaa-e72721e8d306", "ready": false}
2026-04-24T06:31:55.563Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:31:55.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "pod": "some-name-rs0-0"}
2026-04-24T06:31:55.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "pod": "some-name-rs0-1"}
2026-04-24T06:31:55.564Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "pod": "some-name-rs0-2"}
2026-04-24T06:31:55.564Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:31:55.564Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "32731315-04df-4995-b462-7c8767426e0e", "ready": false}
2026-04-24T06:32:00.564Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:32:00.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "pod": "some-name-rs0-1"}
2026-04-24T06:32:00.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "pod": "some-name-rs0-2"}
2026-04-24T06:32:00.565Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "pod": "some-name-rs0-0"}
2026-04-24T06:32:00.565Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:32:00.565Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "b0f2c80f-a7a9-4e94-ac29-f11797aec227", "ready": false}
2026-04-24T06:32:05.566Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:32:05.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "pod": "some-name-rs0-2"}
2026-04-24T06:32:05.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "pod": "some-name-rs0-0"}
2026-04-24T06:32:05.567Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "pod": "some-name-rs0-1"}
2026-04-24T06:32:05.567Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:32:05.567Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c74070d2-e6ba-45c2-9ff2-e4c3d599daa8", "ready": false}
2026-04-24T06:32:10.568Z	DEBUG	checking if restore is allowed	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "cluster": "some-name", "namespace": "non-voting-and-hidden-5889"}
2026-04-24T06:32:10.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "pod": "some-name-rs0-1"}
2026-04-24T06:32:10.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "pod": "some-name-rs0-2"}
2026-04-24T06:32:10.569Z	DEBUG	Pod is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "pod": "some-name-rs0-0"}
2026-04-24T06:32:10.569Z	DEBUG	Statefulset is ready for physical restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "sts": "some-name-rs0", "replset": "rs0"}
2026-04-24T06:32:10.569Z	INFO	Waiting for statefulsets to be ready before restore	{"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"}, "namespace": "non-voting-and-hidden-5889", "name": "restore-backup-physical", "reconcileID": "c1cc1bac-0c3f-4d21-9601-dc7612abd94d", "ready": false}
+ kubectl_bin get psmdb-restore restore-backup-physical -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.pCtuqlRGcU
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZVEF6eRxXI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb-restore restore-backup-physical -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pCtuqlRGcU
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDBRestore
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-physical","namespace":"non-voting-and-hidden-5889"},"spec":{"backupName":"backup-physical","clusterName":"some-name"}}
  creationTimestamp: "2026-04-24T06:02:33Z"
  generation: 1
  name: restore-backup-physical
  namespace: non-voting-and-hidden-5889
  resourceVersion: "1777010554365231016"
  uid: 1556fe24-9d07-4e56-b0dd-8152bee5e509
spec:
  backupName: backup-physical
  clusterName: some-name
status:
  state: waiting
+ cat /tmp/tmp.ZVEF6eRxXI
+ rm /tmp/tmp.pCtuqlRGcU /tmp/tmp.ZVEF6eRxXI
+ return 0
+ kubectl_bin describe psmdb-restore restore-backup-physical
++ mktemp
+ local LAST_OUT=/tmp/tmp.h8le0tWDkQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.6NwOPj9s7A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl describe psmdb-restore restore-backup-physical
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h8le0tWDkQ
Name:         restore-backup-physical
Namespace:    non-voting-and-hidden-5889
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBRestore
Metadata:
  Creation Timestamp:  2026-04-24T06:02:33Z
  Generation:          1
  Resource Version:    1777010554365231016
  UID:                 1556fe24-9d07-4e56-b0dd-8152bee5e509
Spec:
  Backup Name:   backup-physical
  Cluster Name:  some-name
Status:
  State:  waiting
Events:   <none>
+ cat /tmp/tmp.6NwOPj9s7A
+ rm /tmp/tmp.h8le0tWDkQ /tmp/tmp.6NwOPj9s7A
+ return 0
+ echo 'Restore object restore-backup-physical is in waiting state after 29 minutes.'
Restore object restore-backup-physical is in waiting state after 29 minutes.
+ echo something went wrong with operator or kubernetes cluster
something went wrong with operator or kubernetes cluster
+ exit 1