Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/logs/data-at-rest-encryption.log
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
+ create_infra data-at-rest-encryption-5576
+ local ns=data-at-rest-encryption-5576
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.oBBNPlaKsP
++ mktemp
+ local LAST_ERR=/tmp/tmp.h17NB3w8us
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oBBNPlaKsP
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.h17NB3w8us
+ rm /tmp/tmp.oBBNPlaKsP /tmp/tmp.h17NB3w8us
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.h1MJRgSZkE
++ mktemp
+ local LAST_ERR=/tmp/tmp.EP8wbPRKb4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h1MJRgSZkE
+ cat /tmp/tmp.EP8wbPRKb4
+ rm /tmp/tmp.h1MJRgSZkE /tmp/tmp.EP8wbPRKb4
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.k8P9LXrqip
++ mktemp
+ local LAST_ERR=/tmp/tmp.nQJfLHnT7x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k8P9LXrqip
+ cat /tmp/tmp.nQJfLHnT7x
+ rm /tmp/tmp.k8P9LXrqip /tmp/tmp.nQJfLHnT7x
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.OzyTQMsN75
++ mktemp
+ local LAST_ERR=/tmp/tmp.KBh18VPaC7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OzyTQMsN75
+ cat /tmp/tmp.KBh18VPaC7
+ rm /tmp/tmp.OzyTQMsN75 /tmp/tmp.KBh18VPaC7
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.p7pPcaInEt
++ mktemp
+ local LAST_ERR=/tmp/tmp.z2xyHmv8zS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p7pPcaInEt
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.z2xyHmv8zS
+ rm /tmp/tmp.p7pPcaInEt /tmp/tmp.z2xyHmv8zS
+ return 0
+ check_crd_for_deletion PR-2269-20e73be3
+ local git_tag=PR-2269-20e73be3
++ yq eval .metadata.name
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2269-20e73be3/deploy/crd.yaml
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qequZfz2rl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vwoSWrMMGV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qequZfz2rl
++ cat /tmp/tmp.vwoSWrMMGV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qequZfz2rl
++ cat /tmp/tmp.vwoSWrMMGV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.qequZfz2rl
++ cat /tmp/tmp.vwoSWrMMGV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.qequZfz2rl
++ cat /tmp/tmp.vwoSWrMMGV
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.qequZfz2rl /tmp/tmp.vwoSWrMMGV
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator++ mktemp

-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.wrHVAAA2kf
++ mktemp
+ local LAST_OUT=/tmp/tmp.ifI72udF3Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.ylQnJoCzoZ
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.B0iOXd3mJV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ifI72udF3Q
+ cat /tmp/tmp.B0iOXd3mJV
+ rm /tmp/tmp.ifI72udF3Q /tmp/tmp.B0iOXd3mJV
+ return 0
namespace "data-at-rest-encryption-18313" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wrHVAAA2kf
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.ylQnJoCzoZ
+ rm /tmp/tmp.wrHVAAA2kf /tmp/tmp.ylQnJoCzoZ
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.VSBHdtkqBC
++ mktemp
+ local LAST_ERR=/tmp/tmp.c3RRZ50dGV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VSBHdtkqBC
+ cat /tmp/tmp.c3RRZ50dGV
+ rm /tmp/tmp.VSBHdtkqBC /tmp/tmp.c3RRZ50dGV
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.OWrMfecFOc
++ mktemp
+ local LAST_ERR=/tmp/tmp.19MT518qUg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OWrMfecFOc
namespace/psmdb-operator created
+ cat /tmp/tmp.19MT518qUg
+ rm /tmp/tmp.OWrMfecFOc /tmp/tmp.19MT518qUg
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xatumfmahT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CTRt285wrh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xatumfmahT
++ cat /tmp/tmp.CTRt285wrh
++ rm /tmp/tmp.xatumfmahT /tmp/tmp.CTRt285wrh
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.EzjxdB5J4V
++ mktemp
+ local LAST_ERR=/tmp/tmp.LJXRiunitf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EzjxdB5J4V
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8" modified.
+ cat /tmp/tmp.LJXRiunitf
+ rm /tmp/tmp.EzjxdB5J4V /tmp/tmp.LJXRiunitf
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.IY03y4EL65
++ mktemp
+ local LAST_ERR=/tmp/tmp.uKI0Az0ZcO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IY03y4EL65
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.uKI0Az0ZcO
+ rm /tmp/tmp.IY03y4EL65 /tmp/tmp.uKI0Az0ZcO
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wbMH0xDjt5
++ mktemp
+ local LAST_ERR=/tmp/tmp.fNJ1FK3nN9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wbMH0xDjt5
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.fNJ1FK3nN9
+ rm /tmp/tmp.wbMH0xDjt5 /tmp/tmp.fNJ1FK3nN9
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PzFfhloXXk
++ mktemp
+ local LAST_ERR=/tmp/tmp.f8bvRT9n5d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PzFfhloXXk
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.f8bvRT9n5d
+ rm /tmp/tmp.PzFfhloXXk /tmp/tmp.f8bvRT9n5d
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GSAZiBQMDn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KhRpdwFrdA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GSAZiBQMDn
++ cat /tmp/tmp.KhRpdwFrdA
++ rm /tmp/tmp.GSAZiBQMDn /tmp/tmp.KhRpdwFrdA
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-dfddcf789-d2qrg
+ local pod=percona-server-mongodb-operator-dfddcf789-d2qrg
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-dfddcf789-d2qrg to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p1qbpf469o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1LH9Dw3vlG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p1qbpf469o
++ cat /tmp/tmp.1LH9Dw3vlG
++ rm /tmp/tmp.p1qbpf469o /tmp/tmp.1LH9Dw3vlG
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-dfddcf789-d2qrg
++ mktemp
+ local LAST_OUT=/tmp/tmp.tCqEleQbAr
++ mktemp
+ local LAST_ERR=/tmp/tmp.hIK16qO9Ft
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-dfddcf789-d2qrg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tCqEleQbAr
+ cat /tmp/tmp.hIK16qO9Ft
+ rm /tmp/tmp.tCqEleQbAr /tmp/tmp.hIK16qO9Ft
+ return 0
2026-04-23T09:53:42.922Z	INFO	setup	Manager starting up	{"gitCommit": "20e73be336bc9107fc60a16452f9669066123774", "gitBranch": "PR-2269-20e73be3", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace data-at-rest-encryption-5576
+ local namespace=data-at-rest-encryption-5576
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces data-at-rest-encryption-5576'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces data-at-rest-encryption-5576
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace data-at-rest-encryption-5576 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.ElLEn6rQvN
+ local LAST_OUT=/tmp/tmp.ZsIB0HGTML
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.txR0oXuPvO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.Xxyk7Y3N0W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace data-at-rest-encryption-5576 --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ElLEn6rQvN
+ cat /tmp/tmp.Xxyk7Y3N0W
+ rm /tmp/tmp.ElLEn6rQvN /tmp/tmp.Xxyk7Y3N0W
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZsIB0HGTML
+ cat /tmp/tmp.txR0oXuPvO
+ rm /tmp/tmp.ZsIB0HGTML /tmp/tmp.txR0oXuPvO
+ return 0
+ kubectl_bin wait --for=delete namespace data-at-rest-encryption-5576
++ mktemp
+ local LAST_OUT=/tmp/tmp.ShHnZbGbNp
++ mktemp
+ local LAST_ERR=/tmp/tmp.d8ibM1hMY6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace data-at-rest-encryption-5576
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ShHnZbGbNp
+ cat /tmp/tmp.d8ibM1hMY6
+ rm /tmp/tmp.ShHnZbGbNp /tmp/tmp.d8ibM1hMY6
+ return 0
+ desc 'create namespace data-at-rest-encryption-5576'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace data-at-rest-encryption-5576
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace data-at-rest-encryption-5576
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q3Wu6wSRl7
++ mktemp
+ local LAST_ERR=/tmp/tmp.dpQCGs89BQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace data-at-rest-encryption-5576
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Q3Wu6wSRl7
namespace/data-at-rest-encryption-5576 created
+ cat /tmp/tmp.dpQCGs89BQ
+ rm /tmp/tmp.Q3Wu6wSRl7 /tmp/tmp.dpQCGs89BQ
+ return 0
+ set_kube_ctx data-at-rest-encryption-5576
+ local namespace=data-at-rest-encryption-5576
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1aWlKTxici
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oYEkUpj9Fg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1aWlKTxici
++ cat /tmp/tmp.oYEkUpj9Fg
++ rm /tmp/tmp.1aWlKTxici /tmp/tmp.oYEkUpj9Fg
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8 --namespace=data-at-rest-encryption-5576
++ mktemp
+ local LAST_OUT=/tmp/tmp.nhLwZLu1rB
++ mktemp
+ local LAST_ERR=/tmp/tmp.C3djUEzkPE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8 --namespace=data-at-rest-encryption-5576
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nhLwZLu1rB
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster8" modified.
+ cat /tmp/tmp.C3djUEzkPE
+ rm /tmp/tmp.nhLwZLu1rB /tmp/tmp.C3djUEzkPE
+ return 0
+ deploy_vault
+ local name=vault-service
+ [[ 0 -gt 0 ]]
+ desc 'install Vault vault-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Vault vault-service
-----------------------------------------------------------------------------------

+ helm uninstall vault-service
Error: uninstall: Release not loaded: vault-service: release: not found
+ :
+ helm repo remove hashicorp
"hashicorp" has been removed from your repositories
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" has been added to your repositories
+ destroy_vault vault-service
+ local name=vault-service
+ local vault_ns
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter vault-service
+ vault_ns=
+ desc 'destroy vault'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy vault
-----------------------------------------------------------------------------------

++ kubectl api-resources
++ grep vault
++ awk '{print $1}'
+ '[' -n '' ']'
++ kubectl get clusterrolebinding -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding
clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted
clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted
++ kubectl get clusterrole -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole
clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted
++ kubectl get mutatingwebhookconfiguration -l app.kubernetes.io/instance=vault-service -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{end}'
+ timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg
mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted
+ [[ -n '' ]]
+ retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false
NAME: vault-service
LAST DEPLOYED: Thu Apr 23 09:54:27 2026
NAMESPACE: data-at-rest-encryption-5576
STATUS: deployed
REVISION: 1
NOTES:
Thank you for installing HashiCorp Vault!

Now that you have deployed Vault, you should look over the docs on using
Vault with Kubernetes available here:

https://developer.hashicorp.com/vault/docs


Your release is named vault-service. To learn more about the release, try:

  $ helm status vault-service
  $ helm get manifest vault-service
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
+ sleep 1
+ kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}'
+ grep Running
Running
+ sleep 5
+ kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json
++ mktemp
+ local LAST_OUT=/tmp/tmp.IFoQm2pvpu
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Ofo3dpL9g
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IFoQm2pvpu
+ cat /tmp/tmp.9Ofo3dpL9g
+ rm /tmp/tmp.IFoQm2pvpu /tmp/tmp.9Ofo3dpL9g
+ return 0
+ local unsealKey
++ jq -r '.unseal_keys_b64[]'
+ unsealKey=xJBvWONmlebS/fi84+3YITbD/y1DJ5C27r9rVExNmVI=
+ local token
++ jq -r .root_token
+ token=hvs.vjaVXgFJTqWcm5V8Bg2kk97y
+ kubectl_bin exec pod/vault-service-0 -- vault operator unseal xJBvWONmlebS/fi84+3YITbD/y1DJ5C27r9rVExNmVI=
++ mktemp
+ local LAST_OUT=/tmp/tmp.N72YrVaQoB
++ mktemp
+ local LAST_ERR=/tmp/tmp.YxiN6Lq4YK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec pod/vault-service-0 -- vault operator unseal xJBvWONmlebS/fi84+3YITbD/y1DJ5C27r9rVExNmVI=
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N72YrVaQoB
Key             Value
---             -----
Seal Type       shamir
Initialized     true
Sealed          false
Total Shares    1
Threshold       1
Version         1.21.2
Build Date      2026-01-06T08:33:05Z
Storage Type    file
Cluster Name    vault-cluster-357a76fc
Cluster ID      9efd07cb-aab4-5431-de62-fba821eac8a9
HA Enabled      false
+ cat /tmp/tmp.YxiN6Lq4YK
+ rm /tmp/tmp.N72YrVaQoB /tmp/tmp.YxiN6Lq4YK
+ return 0
+ kubectl_bin exec -it pod/vault-service-0 -- sh
++ mktemp
+ local LAST_OUT=/tmp/tmp.VRrTBWmhB6
++ mktemp
+ local LAST_ERR=/tmp/tmp.iceacYQ9vp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec -it pod/vault-service-0 -- sh
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VRrTBWmhB6
Success! You are now authenticated. The token information displayed below
is already stored in the token helper. You do NOT need to run "vault login"
again. Future Vault requests will automatically use this token.

Key                  Value
---                  -----
token                hvs.vjaVXgFJTqWcm5V8Bg2kk97y
token_accessor       8riG5h21wUAxGiMX7rX9xfE6
token_duration       ∞
token_renewable      false
token_policies       ["root"]
identity_policies    []
policies             ["root"]
Success! Enabled the kv-v2 secrets engine at: secret/
+ cat /tmp/tmp.iceacYQ9vp
Unable to use a TTY - input is not a terminal or the right kind of file
+ rm /tmp/tmp.VRrTBWmhB6 /tmp/tmp.iceacYQ9vp
+ return 0
+ kubectl_bin create secret generic vault-secret --from-literal=token=hvs.vjaVXgFJTqWcm5V8Bg2kk97y
++ mktemp
+ local LAST_OUT=/tmp/tmp.VfcMbyOCOv
++ mktemp
+ local LAST_ERR=/tmp/tmp.u0sZl1HJJv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create secret generic vault-secret --from-literal=token=hvs.vjaVXgFJTqWcm5V8Bg2kk97y
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VfcMbyOCOv
secret/vault-secret created
+ cat /tmp/tmp.u0sZl1HJJv
+ rm /tmp/tmp.VfcMbyOCOv /tmp/tmp.u0sZl1HJJv
+ return 0
+ deploy_minio
+ local cert_secret=
+ local service_name=minio-service
+ desc 'install MinIO: minio-service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install MinIO: minio-service
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ local endpoint=http://minio-service:9000
+ minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa')
+ local minio_args
+ [[ -n '' ]]
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Apr 23 09:55:04 2026
NAMESPACE: data-at-rest-encryption-5576
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.data-at-rest-encryption-5576.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace data-at-rest-encryption-5576 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace data-at-rest-encryption-5576

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace data-at-rest-encryption-5576 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace data-at-rest-encryption-5576 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.59HTOrXsi9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s4mBOKiU43
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.59HTOrXsi9
++ cat /tmp/tmp.s4mBOKiU43
++ rm /tmp/tmp.59HTOrXsi9 /tmp/tmp.s4mBOKiU43
++ return 0
+ local MINIO_POD=minio-service-6d5f646cdc-gknmj
+ wait_pod minio-service-6d5f646cdc-gknmj
+ local pod=minio-service-6d5f646cdc-gknmj
+ set +o xtrace
waiting for pod/minio-service-6d5f646cdc-gknmj to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-5576.svc.cluster.local --tcp=9000
service/minio-service created
+ create_minio_bucket operator-testing http://minio-service:9000
+ local bucket=operator-testing
+ local endpoint=http://minio-service:9000
+ kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LysiNdqXBu
++ mktemp
+ local LAST_ERR=/tmp/tmp.6nmK8B71qP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LysiNdqXBu
make_bucket: operator-testing
pod "aws-cli" deleted from data-at-rest-encryption-5576 namespace
+ cat /tmp/tmp.6nmK8B71qP
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.LysiNdqXBu /tmp/tmp.6nmK8B71qP
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nH6AwX4BJt
++ mktemp
+ local LAST_ERR=/tmp/tmp.QZKh7Fl2zf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nH6AwX4BJt
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.QZKh7Fl2zf
+ rm /tmp/tmp.nH6AwX4BJt /tmp/tmp.QZKh7Fl2zf
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.MbziHwr6er
++ mktemp
+ local LAST_ERR=/tmp/tmp.ks3AGSm2Qj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MbziHwr6er
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.ks3AGSm2Qj
+ rm /tmp/tmp.MbziHwr6er /tmp/tmp.ks3AGSm2Qj
+ return 0
+ cluster=some-name
+ desc 'create PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ local LAST_OUT=/tmp/tmp.4dqxgWiZNV
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/data-at-rest-encryption-5576/g
+ local LAST_ERR=/tmp/tmp.kKdSTaax7O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4dqxgWiZNV
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.kKdSTaax7O
+ rm /tmp/tmp.4dqxgWiZNV /tmp/tmp.kKdSTaax7O
+ return 0
+ desc 'check if all Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3 false
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fzok57L2dh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CaHzx06Qo9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Fzok57L2dh
++ cat /tmp/tmp.CaHzx06Qo9
++ rm /tmp/tmp.Fzok57L2dh /tmp/tmp.CaHzx06Qo9
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Y7dkHFTGjE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7CmE4BSBcX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Y7dkHFTGjE
++ cat /tmp/tmp.7CmE4BSBcX
++ rm /tmp/tmp.Y7dkHFTGjE /tmp/tmp.7CmE4BSBcX
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VByBN63oRb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UuJ9YEyVry
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VByBN63oRb
++ cat /tmp/tmp.UuJ9YEyVry
++ rm /tmp/tmp.VByBN63oRb /tmp/tmp.UuJ9YEyVry
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2WloTMCkOT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cQdTJdtoO7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2WloTMCkOT
++ cat /tmp/tmp.cQdTJdtoO7
++ rm /tmp/tmp.2WloTMCkOT /tmp/tmp.cQdTJdtoO7
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QxKPmXuH5g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ir95GaGzuD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QxKPmXuH5g
++ cat /tmp/tmp.ir95GaGzuD
++ rm /tmp/tmp.QxKPmXuH5g /tmp/tmp.ir95GaGzuD
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rMc9gb2aTI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z7rMSH78b5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rMc9gb2aTI
++ cat /tmp/tmp.z7rMSH78b5
++ rm /tmp/tmp.rMc9gb2aTI /tmp/tmp.z7rMSH78b5
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zznIIP0Wko
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TjytAJID17
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zznIIP0Wko
++ cat /tmp/tmp.TjytAJID17
++ rm /tmp/tmp.zznIIP0Wko /tmp/tmp.TjytAJID17
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zRFk6jtgiG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zy8JjE2W78
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zRFk6jtgiG
++ cat /tmp/tmp.Zy8JjE2W78
++ rm /tmp/tmp.zRFk6jtgiG /tmp/tmp.Zy8JjE2W78
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qWH87jd5sx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d6axnjzqZ7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qWH87jd5sx
++ cat /tmp/tmp.d6axnjzqZ7
++ rm /tmp/tmp.qWH87jd5sx /tmp/tmp.d6axnjzqZ7
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.Y3AvjVzNij/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-5576", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6w7JWBeJkn
++ mktemp
+ local LAST_ERR=/tmp/tmp.7Qeb9ZxHgY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6w7JWBeJkn
+ cat /tmp/tmp.7Qeb9ZxHgY
+ rm /tmp/tmp.6w7JWBeJkn /tmp/tmp.7Qeb9ZxHgY
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml /tmp/tmp.Y3AvjVzNij/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-23T09:58:23+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.Y3AvjVzNij/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-5576", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SxSFaFiDEO
++ mktemp
+ local LAST_ERR=/tmp/tmp.WLiUQ7Ci94
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SxSFaFiDEO
+ cat /tmp/tmp.WLiUQ7Ci94
+ rm /tmp/tmp.SxSFaFiDEO /tmp/tmp.WLiUQ7Ci94
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Y3AvjVzNij/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml /tmp/tmp.Y3AvjVzNij/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-04-23T09:58:24+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-5576
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-5576
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yTIlKajGTV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Lx8a1VLXG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yTIlKajGTV
++ cat /tmp/tmp.4Lx8a1VLXG
++ rm /tmp/tmp.yTIlKajGTV /tmp/tmp.4Lx8a1VLXG
++ return 0
+ local client_container=psmdb-client-bb8b97679-cbss4
+ kubectl_bin exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.peD04xV1kL
++ mktemp
+ local LAST_ERR=/tmp/tmp.1L9sLj5fiL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.peD04xV1kL
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d1ce24d4-5164-45fe-8682-3acbbc6252cb") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.1L9sLj5fiL
+ rm /tmp/tmp.peD04xV1kL /tmp/tmp.1L9sLj5fiL
+ return 0
+ sleep 2
+ desc 'write data, read it'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read it
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.data-at-rest-encryption-5576
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-5576
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HMvSkZynpn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1cIyWGFhVz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HMvSkZynpn
++ cat /tmp/tmp.1cIyWGFhVz
++ rm /tmp/tmp.HMvSkZynpn /tmp/tmp.1cIyWGFhVz
++ return 0
+ local client_container=psmdb-client-bb8b97679-cbss4
+ kubectl_bin exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.I1JvR1cqkB
++ mktemp
+ local LAST_ERR=/tmp/tmp.3Dn7yUlp9b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I1JvR1cqkB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("666d7b06-7dcc-4d41-bd76-b84ba26b345b") }
Percona Server for MongoDB server version: v8.0.20-8
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.3Dn7yUlp9b
+ rm /tmp/tmp.I1JvR1cqkB /tmp/tmp.3Dn7yUlp9b
+ return 0
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.data-at-rest-encryption-5576
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-5576
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ local tls=false
+ [[ false == true ]]
+ mongos_command=run_mongos
+ log 'running db.test.command() in myApp'
+ set +o xtrace
[2026-04-23T09:58:32+0000] running db.test.command() in myApp
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.data-at-rest-encryption-5576 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-5576
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iMzGfo5x6S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jkoMHxFigv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iMzGfo5x6S
++ cat /tmp/tmp.jkoMHxFigv
++ rm /tmp/tmp.iMzGfo5x6S /tmp/tmp.jkoMHxFigv
++ return 0
+ local client_container=psmdb-client-bb8b97679-cbss4
+ kubectl_bin exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aFcU2IFz2p
++ mktemp
+ local LAST_ERR=/tmp/tmp.Amjnoo14sM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-cbss4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-5576.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aFcU2IFz2p
+ cat /tmp/tmp.Amjnoo14sM
+ rm /tmp/tmp.aFcU2IFz2p /tmp/tmp.Amjnoo14sM
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/compare/find.json /tmp/tmp.Y3AvjVzNij/find
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ backup_name_minio=backup-minio
+ backup_name_gcp=backup-gcp-cs
+ desc 'run minio backup backup-minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
run minio backup backup-minio
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ log 'running backup backup-minio'
+ set +o xtrace
[2026-04-23T09:58:34+0000] running backup backup-minio
+ yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/data-at-rest-encryption/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.X4tNFzVXZ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.ii45JOeeEB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X4tNFzVXZ5
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.ii45JOeeEB
+ rm /tmp/tmp.X4tNFzVXZ5 /tmp/tmp.ii45JOeeEB
+ return 0
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:03.722Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "1cb095a6-cd03-4267-9252-e7f7afd62b5a"}
2026-04-23T10:18:04.594Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "21a68491-7a06-480b-ba28-d5e3e6a98fef", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:04.595Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "21a68491-7a06-480b-ba28-d5e3e6a98fef", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:04.993Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "21a68491-7a06-480b-ba28-d5e3e6a98fef", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:06.508Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "21a68491-7a06-480b-ba28-d5e3e6a98fef", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:08.723Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "6835d5ec-6722-48e4-b545-e4fe02e7c897"}
2026-04-23T10:18:08.735Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "6835d5ec-6722-48e4-b545-e4fe02e7c897", "state": "requested"}
2026-04-23T10:18:08.812Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "6835d5ec-6722-48e4-b545-e4fe02e7c897", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:08.812Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "6835d5ec-6722-48e4-b545-e4fe02e7c897", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:08.813Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "6835d5ec-6722-48e4-b545-e4fe02e7c897"}
2026-04-23T10:18:13.770Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0357b37a-0077-44a8-a1e2-14cf7106edbb", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:13.771Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0357b37a-0077-44a8-a1e2-14cf7106edbb", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:13.814Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "819bcac0-3c53-4d81-bb78-42b60aee04de"}
2026-04-23T10:18:13.825Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "819bcac0-3c53-4d81-bb78-42b60aee04de", "state": "requested"}
2026-04-23T10:18:13.916Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "819bcac0-3c53-4d81-bb78-42b60aee04de", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:13.916Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "819bcac0-3c53-4d81-bb78-42b60aee04de", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:13.917Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "819bcac0-3c53-4d81-bb78-42b60aee04de"}
2026-04-23T10:18:14.395Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0357b37a-0077-44a8-a1e2-14cf7106edbb", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:16.223Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0357b37a-0077-44a8-a1e2-14cf7106edbb", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:18.919Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "fc4f0b74-1188-41f0-8f17-e887bfbfc9cb"}
2026-04-23T10:18:18.938Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "fc4f0b74-1188-41f0-8f17-e887bfbfc9cb", "state": "requested"}
2026-04-23T10:18:19.029Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "fc4f0b74-1188-41f0-8f17-e887bfbfc9cb", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:19.029Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "fc4f0b74-1188-41f0-8f17-e887bfbfc9cb", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:19.030Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "fc4f0b74-1188-41f0-8f17-e887bfbfc9cb"}
2026-04-23T10:18:23.699Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0b0f4989-1263-469f-92b0-d94be873a905", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:23.700Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0b0f4989-1263-469f-92b0-d94be873a905", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:24.031Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "144ce00d-f1a3-4ee3-803c-d557f23487e1"}
2026-04-23T10:18:24.049Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "144ce00d-f1a3-4ee3-803c-d557f23487e1", "state": "requested"}
2026-04-23T10:18:24.105Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0b0f4989-1263-469f-92b0-d94be873a905", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:24.155Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "144ce00d-f1a3-4ee3-803c-d557f23487e1", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:24.155Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "144ce00d-f1a3-4ee3-803c-d557f23487e1", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:24.156Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "144ce00d-f1a3-4ee3-803c-d557f23487e1"}
2026-04-23T10:18:25.610Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "0b0f4989-1263-469f-92b0-d94be873a905", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:29.157Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "79e54b15-f784-4332-95e8-c6f9666d79e0"}
2026-04-23T10:18:29.172Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "79e54b15-f784-4332-95e8-c6f9666d79e0", "state": "requested"}
2026-04-23T10:18:29.257Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "79e54b15-f784-4332-95e8-c6f9666d79e0", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:29.257Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "79e54b15-f784-4332-95e8-c6f9666d79e0", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:29.258Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "79e54b15-f784-4332-95e8-c6f9666d79e0"}
2026-04-23T10:18:33.044Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "8c21387d-28b7-403c-bad0-ad220c4bb1e9", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:33.045Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "8c21387d-28b7-403c-bad0-ad220c4bb1e9", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:33.455Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "8c21387d-28b7-403c-bad0-ad220c4bb1e9", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:34.259Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "b295747f-7161-405f-8e65-d8074211fe73"}
2026-04-23T10:18:34.270Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "b295747f-7161-405f-8e65-d8074211fe73", "state": "requested"}
2026-04-23T10:18:34.355Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "b295747f-7161-405f-8e65-d8074211fe73", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:34.355Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "b295747f-7161-405f-8e65-d8074211fe73", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:34.356Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "b295747f-7161-405f-8e65-d8074211fe73"}
2026-04-23T10:18:34.947Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "8c21387d-28b7-403c-bad0-ad220c4bb1e9", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:39.357Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f03c0943-ef90-4c53-a0a0-eae67215b145"}
2026-04-23T10:18:39.369Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f03c0943-ef90-4c53-a0a0-eae67215b145", "state": "requested"}
2026-04-23T10:18:39.454Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f03c0943-ef90-4c53-a0a0-eae67215b145", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:39.454Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f03c0943-ef90-4c53-a0a0-eae67215b145", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:39.455Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f03c0943-ef90-4c53-a0a0-eae67215b145"}
2026-04-23T10:18:42.227Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "092ed21f-5cce-45ec-afe3-101a060c533d", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:42.228Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "092ed21f-5cce-45ec-afe3-101a060c533d", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:42.626Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "092ed21f-5cce-45ec-afe3-101a060c533d", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:44.100Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "092ed21f-5cce-45ec-afe3-101a060c533d", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:44.457Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "929b1028-ea1f-47a4-9c44-2b1704ec9281"}
2026-04-23T10:18:44.483Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "929b1028-ea1f-47a4-9c44-2b1704ec9281", "state": "requested"}
2026-04-23T10:18:44.559Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "929b1028-ea1f-47a4-9c44-2b1704ec9281", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:44.559Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "929b1028-ea1f-47a4-9c44-2b1704ec9281", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:44.560Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "929b1028-ea1f-47a4-9c44-2b1704ec9281"}
2026-04-23T10:18:49.560Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "15f3a645-928b-4aa5-8300-27348013d4bb"}
2026-04-23T10:18:49.572Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "15f3a645-928b-4aa5-8300-27348013d4bb", "state": "requested"}
2026-04-23T10:18:49.654Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "15f3a645-928b-4aa5-8300-27348013d4bb", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:49.655Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "15f3a645-928b-4aa5-8300-27348013d4bb", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:49.656Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "15f3a645-928b-4aa5-8300-27348013d4bb"}
2026-04-23T10:18:51.518Z	DEBUG	Replset is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "aa2a6e83-27b5-4a3a-8dd0-cf22ceaa480c", "liveMembers": 2, "pods": 3}
2026-04-23T10:18:51.519Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "aa2a6e83-27b5-4a3a-8dd0-cf22ceaa480c", "replset": "cfg", "status": "initializing"}
2026-04-23T10:18:51.912Z	DEBUG	Replset status is not healthy	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "aa2a6e83-27b5-4a3a-8dd0-cf22ceaa480c", "replset": "rs0", "status": "initializing"}
2026-04-23T10:18:53.524Z	DEBUG	Cluster is not ready	{"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "some-name", "reconcileID": "aa2a6e83-27b5-4a3a-8dd0-cf22ceaa480c", "pbmStatus": "ready", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"}
2026-04-23T10:18:54.656Z	DEBUG	Reconciling	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f94f5298-eae0-4895-aa76-ef52f2221bf6"}
2026-04-23T10:18:54.671Z	DEBUG	Got object from API server	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f94f5298-eae0-4895-aa76-ef52f2221bf6", "state": "requested"}
2026-04-23T10:18:54.751Z	DEBUG	checking if backup is allowed	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f94f5298-eae0-4895-aa76-ef52f2221bf6", "cluster": "some-name", "namespace": "data-at-rest-encryption-5576"}
2026-04-23T10:18:54.751Z	ERROR	Cluster is not ready for backup	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f94f5298-eae0-4895-aa76-ef52f2221bf6", "error": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status initializing\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1631\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:259\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:260
github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile
	/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/psmdb_backup_controller.go:237
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
2026-04-23T10:18:54.752Z	DEBUG	Reconcile finished	{"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-5576"}, "namespace": "data-at-rest-encryption-5576", "name": "backup-minio", "reconcileID": "f94f5298-eae0-4895-aa76-ef52f2221bf6"}
NAME           CLUSTER     STORAGE   DESTINATION                                  TYPE   SIZE   STATUS      COMPLETED   AGE
backup-minio   some-name   minio     s3://operator-testing/2026-04-23T09:58:37Z                 requested               20m
Name:         backup-minio
Namespace:    data-at-rest-encryption-5576
Labels:       <none>
Annotations:  <none>
API Version:  psmdb.percona.com/v1
Kind:         PerconaServerMongoDBBackup
Metadata:
  Creation Timestamp:  2026-04-23T09:58:37Z
  Finalizers:
    internal.percona.com/release-lock
  Generation:        1
  Resource Version:  1776938317787999011
  UID:               2ea21f70-5cbe-4a84-922b-ec13363cb354
Spec:
  Cluster Name:               some-name
  Starting Deadline Seconds:  120
  Storage Name:               minio
  Type:                       logical
Status:
  Destination:      s3://operator-testing/2026-04-23T09:58:37Z
  Last Transition:  2026-04-23T09:58:37Z
  Pbm Name:         2026-04-23T09:58:37Z
  Replset Names:
    cfg
    rs0
  s3:
    Bucket:              operator-testing
    Credentials Secret:  minio-secret
    Endpoint URL:        http://minio-service:9000/
    Region:              us-east-1
    Server Side Encryption:
  State:         requested
  Storage Name:  minio
Events:          <none>
Backup object psmdb-backup/backup-minio is in requested state.