Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/logs/data-at-rest-encryption.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + create_infra data-at-rest-encryption-21488 + local ns=data-at-rest-encryption-21488 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.ElYzBZHSZW ++ mktemp + local LAST_ERR=/tmp/tmp.7Jgt7GxxsA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ElYzBZHSZW customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.7Jgt7GxxsA + rm /tmp/tmp.ElYzBZHSZW /tmp/tmp.7Jgt7GxxsA + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.HmfZChsKXB ++ mktemp + local LAST_ERR=/tmp/tmp.8vfXPNKknV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HmfZChsKXB customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.8vfXPNKknV + rm /tmp/tmp.HmfZChsKXB /tmp/tmp.8vfXPNKknV + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.LrnBQgHtlp ++ mktemp + local LAST_ERR=/tmp/tmp.a0llTjOBsP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LrnBQgHtlp + cat /tmp/tmp.a0llTjOBsP + rm /tmp/tmp.LrnBQgHtlp /tmp/tmp.a0llTjOBsP + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.gfYfVS6VWH ++ mktemp + local LAST_ERR=/tmp/tmp.GNC5L16V88 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gfYfVS6VWH + cat /tmp/tmp.GNC5L16V88 + rm /tmp/tmp.gfYfVS6VWH /tmp/tmp.GNC5L16V88 + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.LI1Tt88fLM ++ mktemp + local LAST_ERR=/tmp/tmp.g8fQOrrAMM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LI1Tt88fLM clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.g8fQOrrAMM + rm /tmp/tmp.LI1Tt88fLM /tmp/tmp.g8fQOrrAMM + return 0 + check_crd_for_deletion PR-1987-3dea319b + local git_tag=PR-1987-3dea319b ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1987-3dea319b/deploy/crd.yaml ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/bin/sed s/---//g ++ yq eval .metadata.name + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hRQ3Wwnlll +++ mktemp ++ local LAST_ERR=/tmp/tmp.9zENpLtifJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hRQ3Wwnlll ++ cat /tmp/tmp.9zENpLtifJ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hRQ3Wwnlll ++ cat /tmp/tmp.9zENpLtifJ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.hRQ3Wwnlll ++ cat /tmp/tmp.9zENpLtifJ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.hRQ3Wwnlll ++ cat /tmp/tmp.9zENpLtifJ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.hRQ3Wwnlll /tmp/tmp.9zENpLtifJ ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.hhecRjQeRE + local LAST_OUT=/tmp/tmp.mnXTVBBzQ7 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.aNucV0tFrf + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.5zIGCsYvT5 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + for i in '$(seq 0 2)' + set +e + set +e + kubectl get ns + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hhecRjQeRE + cat /tmp/tmp.aNucV0tFrf + rm /tmp/tmp.hhecRjQeRE /tmp/tmp.aNucV0tFrf + return 0 namespace "data-at-rest-encryption-19455" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mnXTVBBzQ7 namespace "psmdb-operator" deleted + cat /tmp/tmp.5zIGCsYvT5 + rm /tmp/tmp.mnXTVBBzQ7 /tmp/tmp.5zIGCsYvT5 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.cJs17kL3FE ++ mktemp + local LAST_ERR=/tmp/tmp.CFwQ4nxpqR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cJs17kL3FE + cat /tmp/tmp.CFwQ4nxpqR + rm /tmp/tmp.cJs17kL3FE /tmp/tmp.CFwQ4nxpqR + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.tlSi8Fp0qc ++ mktemp + local LAST_ERR=/tmp/tmp.I33h3U10Ez + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tlSi8Fp0qc namespace/psmdb-operator created + cat /tmp/tmp.I33h3U10Ez + rm /tmp/tmp.tlSi8Fp0qc /tmp/tmp.I33h3U10Ez + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.8oqhHCF6Th +++ mktemp ++ local LAST_ERR=/tmp/tmp.KqFevOPomP ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8oqhHCF6Th ++ cat /tmp/tmp.KqFevOPomP ++ rm /tmp/tmp.8oqhHCF6Th /tmp/tmp.KqFevOPomP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.VVazU9u84K ++ mktemp + local LAST_ERR=/tmp/tmp.TzXjUnY4om + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VVazU9u84K Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8" modified. + cat /tmp/tmp.TzXjUnY4om + rm /tmp/tmp.VVazU9u84K /tmp/tmp.TzXjUnY4om + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.D5UyfXgbti ++ mktemp + local LAST_ERR=/tmp/tmp.lJUfhqs1PQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D5UyfXgbti customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.lJUfhqs1PQ + rm /tmp/tmp.D5UyfXgbti /tmp/tmp.lJUfhqs1PQ + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.W9tu3XdY2p ++ mktemp + local LAST_ERR=/tmp/tmp.zuJmpeP88E + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W9tu3XdY2p clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.zuJmpeP88E + rm /tmp/tmp.W9tu3XdY2p /tmp/tmp.zuJmpeP88E + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1987-3dea319b") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AVEDrbcz3K ++ mktemp + local LAST_ERR=/tmp/tmp.sTXiNUDu5n + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AVEDrbcz3K deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.sTXiNUDu5n + rm /tmp/tmp.AVEDrbcz3K /tmp/tmp.sTXiNUDu5n + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.FCzh5bpdLJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.1RPWGDAMYV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FCzh5bpdLJ ++ cat /tmp/tmp.1RPWGDAMYV ++ rm /tmp/tmp.FCzh5bpdLJ /tmp/tmp.1RPWGDAMYV ++ return 0 + wait_pod percona-server-mongodb-operator-76947877-grgk2 + local pod=percona-server-mongodb-operator-76947877-grgk2 + set +o xtrace waiting for pod/percona-server-mongodb-operator-76947877-grgk2 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y7AkT6HlGL +++ mktemp ++ local LAST_ERR=/tmp/tmp.OV9F4bEH0w ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Y7AkT6HlGL ++ cat /tmp/tmp.OV9F4bEH0w ++ rm /tmp/tmp.Y7AkT6HlGL /tmp/tmp.OV9F4bEH0w ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-76947877-grgk2 ++ mktemp + local LAST_OUT=/tmp/tmp.tdOnx2DtOc ++ mktemp + local LAST_ERR=/tmp/tmp.PHncvxnoAc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-76947877-grgk2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tdOnx2DtOc + cat /tmp/tmp.PHncvxnoAc + rm /tmp/tmp.tdOnx2DtOc /tmp/tmp.PHncvxnoAc + return 0 2025-07-01T08:46:26.004Z INFO setup Manager starting up {"gitCommit": "3dea319bf7c889a0f47b7fdd53cac66e613a6947", "gitBranch": "PR-1987-3dea319b", "buildTime": "", "goVersion": "go1.24.4", "os": "linux", "arch": "amd64"} + create_namespace data-at-rest-encryption-21488 + local namespace=data-at-rest-encryption-21488 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces data-at-rest-encryption-21488' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces data-at-rest-encryption-21488 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace data-at-rest-encryption-21488 --ignore-not-found + kubectl_bin get ns ++ mktemp ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.ilDnMFw0sc ++ mktemp + local LAST_OUT=/tmp/tmp.afVPxX8Kh9 ++ mktemp + local LAST_ERR=/tmp/tmp.VQxjJ0ZSp8 + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.uVI1ubfNV0 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace data-at-rest-encryption-21488 --ignore-not-found + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.afVPxX8Kh9 + cat /tmp/tmp.uVI1ubfNV0 + rm /tmp/tmp.afVPxX8Kh9 /tmp/tmp.uVI1ubfNV0 + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ilDnMFw0sc + cat /tmp/tmp.VQxjJ0ZSp8 + rm /tmp/tmp.ilDnMFw0sc /tmp/tmp.VQxjJ0ZSp8 + return 0 + kubectl_bin wait --for=delete namespace data-at-rest-encryption-21488 ++ mktemp + local LAST_OUT=/tmp/tmp.utPNLvc0M7 ++ mktemp + local LAST_ERR=/tmp/tmp.fBr7agCsVb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace data-at-rest-encryption-21488 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.utPNLvc0M7 + cat /tmp/tmp.fBr7agCsVb + rm /tmp/tmp.utPNLvc0M7 /tmp/tmp.fBr7agCsVb + return 0 + desc 'create namespace data-at-rest-encryption-21488' + set +o xtrace ----------------------------------------------------------------------------------- create namespace data-at-rest-encryption-21488 ----------------------------------------------------------------------------------- + kubectl_bin create namespace data-at-rest-encryption-21488 ++ mktemp + local LAST_OUT=/tmp/tmp.lCN3wbd7HJ ++ mktemp + local LAST_ERR=/tmp/tmp.UfZCuSefHA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace data-at-rest-encryption-21488 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lCN3wbd7HJ namespace/data-at-rest-encryption-21488 created + cat /tmp/tmp.UfZCuSefHA + rm /tmp/tmp.lCN3wbd7HJ /tmp/tmp.UfZCuSefHA + return 0 + set_kube_ctx data-at-rest-encryption-21488 + local namespace=data-at-rest-encryption-21488 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.zZpGvSc7DM +++ mktemp ++ local LAST_ERR=/tmp/tmp.uJIIQDKHpG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zZpGvSc7DM ++ cat /tmp/tmp.uJIIQDKHpG ++ rm /tmp/tmp.zZpGvSc7DM /tmp/tmp.uJIIQDKHpG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8 --namespace=data-at-rest-encryption-21488 ++ mktemp + local LAST_OUT=/tmp/tmp.BxuP7ZB8jK ++ mktemp + local LAST_ERR=/tmp/tmp.yJ3dBhLFJe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8 --namespace=data-at-rest-encryption-21488 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BxuP7ZB8jK Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1987-3dea319b-2-cluster8" modified. + cat /tmp/tmp.yJ3dBhLFJe + rm /tmp/tmp.BxuP7ZB8jK /tmp/tmp.yJ3dBhLFJe + return 0 + deploy_vault + name=vault-service + desc 'install Vault vault-service' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service ----------------------------------------------------------------------------------- + helm uninstall vault-service Error: uninstall: Release not loaded: vault-service: release: not found + : + helm repo remove hashicorp "hashicorp" has been removed from your repositories + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" has been added to your repositories + destroy_vault ++ helm list --all-namespaces --filter vault-service ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local vault_ns= + desc 'destroy vault' + set +o xtrace ----------------------------------------------------------------------------------- destroy vault ----------------------------------------------------------------------------------- ++ kubectl api-resources ++ grep vault ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted ++ kubectl get clusterrole ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted ++ kubectl get mutatingwebhookconfiguration ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted + [[ -n '' ]] + retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false + local max=10 + local delay=60 + shift 2 + local n=1 + helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false NAME: vault-service LAST DEPLOYED: Tue Jul 1 08:46:51 2025 NAMESPACE: data-at-rest-encryption-21488 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service. To learn more about the release, try: $ helm status vault-service $ helm get manifest vault-service + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running Running + kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.0T9uVgeSml ++ mktemp + local LAST_ERR=/tmp/tmp.3u3VTz0NHf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0T9uVgeSml + cat /tmp/tmp.3u3VTz0NHf + rm /tmp/tmp.0T9uVgeSml /tmp/tmp.3u3VTz0NHf + return 0 ++ jq -r '.unseal_keys_b64[]' + local unsealKey=Z098LBjBqciOfsXRdCCPZBnUa5WicRGTUqr+i44Jgbc= ++ jq -r .root_token + local token=hvs.lWgUERm3WriTNW5aD9SNTgCO + kubectl_bin exec pod/vault-service-0 -- vault operator unseal Z098LBjBqciOfsXRdCCPZBnUa5WicRGTUqr+i44Jgbc= ++ mktemp + local LAST_OUT=/tmp/tmp.0GSEeTjXZm ++ mktemp + local LAST_ERR=/tmp/tmp.kneqvGfdp0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator unseal Z098LBjBqciOfsXRdCCPZBnUa5WicRGTUqr+i44Jgbc= + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0GSEeTjXZm Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 1.19.0 Build Date 2025-03-04T12:36:40Z Storage Type file Cluster Name vault-cluster-444e7428 Cluster ID 79e0b0f3-2b60-ce90-5a30-4671478ff4ae HA Enabled false + cat /tmp/tmp.kneqvGfdp0 + rm /tmp/tmp.0GSEeTjXZm /tmp/tmp.kneqvGfdp0 + return 0 + kubectl_bin exec -it pod/vault-service-0 -- sh ++ mktemp + local LAST_OUT=/tmp/tmp.bdsh1MVhK7 ++ mktemp + local LAST_ERR=/tmp/tmp.hBbctDkjrk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it pod/vault-service-0 -- sh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bdsh1MVhK7 Success! You are now authenticated. The token information displayed below is already stored in the token helper. You do NOT need to run "vault login" again. Future Vault requests will automatically use this token. Key Value --- ----- token hvs.lWgUERm3WriTNW5aD9SNTgCO token_accessor 4dV4sLR475bvi4R0uYDOiFP9 token_duration ∞ token_renewable false token_policies ["root"] identity_policies [] policies ["root"] Success! Enabled the kv-v2 secrets engine at: secret/ + cat /tmp/tmp.hBbctDkjrk Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.bdsh1MVhK7 /tmp/tmp.hBbctDkjrk + return 0 + kubectl_bin create secret generic vault-secret --from-literal=token=hvs.lWgUERm3WriTNW5aD9SNTgCO ++ mktemp + local LAST_OUT=/tmp/tmp.Mad1pxZ0tm ++ mktemp + local LAST_ERR=/tmp/tmp.CASfo4CFKc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create secret generic vault-secret --from-literal=token=hvs.lWgUERm3WriTNW5aD9SNTgCO + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Mad1pxZ0tm secret/vault-secret created + cat /tmp/tmp.CASfo4CFKc + rm /tmp/tmp.Mad1pxZ0tm /tmp/tmp.CASfo4CFKc + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Tue Jul 1 08:47:17 2025 NAMESPACE: data-at-rest-encryption-21488 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.data-at-rest-encryption-21488.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace data-at-rest-encryption-21488 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace data-at-rest-encryption-21488 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace data-at-rest-encryption-21488 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace data-at-rest-encryption-21488 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TUSxasKvZG +++ mktemp ++ local LAST_ERR=/tmp/tmp.mzcM3HCQRw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TUSxasKvZG ++ cat /tmp/tmp.mzcM3HCQRw ++ rm /tmp/tmp.TUSxasKvZG /tmp/tmp.mzcM3HCQRw ++ return 0 + MINIO_POD=minio-service-86dfccd949-h9jdg + wait_pod minio-service-86dfccd949-h9jdg + local pod=minio-service-86dfccd949-h9jdg + set +o xtrace waiting for pod/minio-service-86dfccd949-h9jdg to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-21488.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.NkcFoq6Mjm ++ mktemp + local LAST_ERR=/tmp/tmp.K0Xcy1lXGP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-21488.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NkcFoq6Mjm service/minio-service created + cat /tmp/tmp.K0Xcy1lXGP + rm /tmp/tmp.NkcFoq6Mjm /tmp/tmp.K0Xcy1lXGP + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.dvg5HrVxmd ++ mktemp + local LAST_ERR=/tmp/tmp.qTzWskF84z + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dvg5HrVxmd pod "aws-cli" deleted + cat /tmp/tmp.qTzWskF84z If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.dvg5HrVxmd /tmp/tmp.qTzWskF84z + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.dHgKxASyFj ++ mktemp + local LAST_ERR=/tmp/tmp.aI2j5hHcRY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dHgKxASyFj secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.aI2j5hHcRY + rm /tmp/tmp.dHgKxASyFj /tmp/tmp.aI2j5hHcRY + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7QbinDNzYw ++ mktemp + local LAST_ERR=/tmp/tmp.Rxft72VZT1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7QbinDNzYw secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.Rxft72VZT1 + rm /tmp/tmp.7QbinDNzYw /tmp/tmp.Rxft72VZT1 + return 0 + cluster=some-name + desc 'create PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1987-3dea319b"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.tnqUjbvGLh ++ mktemp + local LAST_ERR=/tmp/tmp.Z0jchyutHU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tnqUjbvGLh perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.Z0jchyutHU + rm /tmp/tmp.tnqUjbvGLh /tmp/tmp.Z0jchyutHU + return 0 + desc 'check if all Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 false + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready..........OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AEBoJlRy1L +++ mktemp ++ local LAST_ERR=/tmp/tmp.R5xaGsTYC0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AEBoJlRy1L ++ cat /tmp/tmp.R5xaGsTYC0 ++ rm /tmp/tmp.AEBoJlRy1L /tmp/tmp.R5xaGsTYC0 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vbssNpchDj +++ mktemp ++ local LAST_ERR=/tmp/tmp.WV4deL6gSC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vbssNpchDj ++ cat /tmp/tmp.WV4deL6gSC ++ rm /tmp/tmp.vbssNpchDj /tmp/tmp.WV4deL6gSC ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W94xBZHpxa +++ mktemp ++ local LAST_ERR=/tmp/tmp.166dQQPe8L ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W94xBZHpxa ++ cat /tmp/tmp.166dQQPe8L ++ rm /tmp/tmp.W94xBZHpxa /tmp/tmp.166dQQPe8L ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uVtGMDEq1y +++ mktemp ++ local LAST_ERR=/tmp/tmp.mvFwHjnTVt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uVtGMDEq1y ++ cat /tmp/tmp.mvFwHjnTVt ++ rm /tmp/tmp.uVtGMDEq1y /tmp/tmp.mvFwHjnTVt ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jo3vv6xUvl +++ mktemp ++ local LAST_ERR=/tmp/tmp.MZgRVnkE1p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Jo3vv6xUvl ++ cat /tmp/tmp.MZgRVnkE1p ++ rm /tmp/tmp.Jo3vv6xUvl /tmp/tmp.MZgRVnkE1p ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Iwp8mWrbsf +++ mktemp ++ local LAST_ERR=/tmp/tmp.LtYey6kkwc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Iwp8mWrbsf ++ cat /tmp/tmp.LtYey6kkwc ++ rm /tmp/tmp.Iwp8mWrbsf /tmp/tmp.LtYey6kkwc ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.....OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U6XotURRin +++ mktemp ++ local LAST_ERR=/tmp/tmp.PvW4F8xcAO ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U6XotURRin ++ cat /tmp/tmp.PvW4F8xcAO ++ rm /tmp/tmp.U6XotURRin /tmp/tmp.PvW4F8xcAO ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NFk3MJZtKD +++ mktemp ++ local LAST_ERR=/tmp/tmp.GczdNuYt4X ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NFk3MJZtKD ++ cat /tmp/tmp.GczdNuYt4X ++ rm /tmp/tmp.NFk3MJZtKD /tmp/tmp.GczdNuYt4X ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y387feBJ5M +++ mktemp ++ local LAST_ERR=/tmp/tmp.jXKKfawv9D ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Y387feBJ5M ++ cat /tmp/tmp.jXKKfawv9D ++ rm /tmp/tmp.Y387feBJ5M /tmp/tmp.jXKKfawv9D ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.Xzh2TBRSnh/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-21488", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.fcODr2vJ6f ++ mktemp + local LAST_ERR=/tmp/tmp.k8DsuSEduD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fcODr2vJ6f + cat /tmp/tmp.k8DsuSEduD + rm /tmp/tmp.fcODr2vJ6f /tmp/tmp.k8DsuSEduD + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-rs0.yml + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.Xzh2TBRSnh/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-21488", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.UsqjyA4KmP ++ mktemp + local LAST_ERR=/tmp/tmp.9ILB13BNoV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UsqjyA4KmP + cat /tmp/tmp.9ILB13BNoV + rm /tmp/tmp.UsqjyA4KmP /tmp/tmp.9ILB13BNoV + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-cfg.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml /tmp/tmp.Xzh2TBRSnh/statefulset_some-name-cfg.yml + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-21488 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-21488 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FvdXkzEimu +++ mktemp ++ local LAST_ERR=/tmp/tmp.4fkW9x8u5W ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FvdXkzEimu ++ cat /tmp/tmp.4fkW9x8u5W ++ rm /tmp/tmp.FvdXkzEimu /tmp/tmp.4fkW9x8u5W ++ return 0 + local client_container=psmdb-client-66f577db5f-xt92d + kubectl_bin exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Be9rs6ruxm ++ mktemp + local LAST_ERR=/tmp/tmp.UZBp694MhO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Be9rs6ruxm Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("4e7a234b-ddc1-44a5-b0ae-ed36def4fb2f") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.UZBp694MhO + rm /tmp/tmp.Be9rs6ruxm /tmp/tmp.UZBp694MhO + return 0 + sleep 2 + desc 'write data, read it' + set +o xtrace ----------------------------------------------------------------------------------- write data, read it ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vVLfTFlUp3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pm4V6yXARr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vVLfTFlUp3 ++ cat /tmp/tmp.Pm4V6yXARr ++ rm /tmp/tmp.vVLfTFlUp3 /tmp/tmp.Pm4V6yXARr ++ return 0 + local client_container=psmdb-client-66f577db5f-xt92d + kubectl_bin exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.FDeFM4p7h6 ++ mktemp + local LAST_ERR=/tmp/tmp.AY5N98r7YE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FDeFM4p7h6 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("d73377d8-0ae5-40a4-92db-dfeb1335bec3") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.AY5N98r7YE + rm /tmp/tmp.FDeFM4p7h6 /tmp/tmp.AY5N98r7YE + return 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 + local command=find + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-21488 + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vpTYCZuzcS +++ mktemp ++ local LAST_ERR=/tmp/tmp.6TFh6JmQk0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vpTYCZuzcS ++ cat /tmp/tmp.6TFh6JmQk0 ++ rm /tmp/tmp.vpTYCZuzcS /tmp/tmp.6TFh6JmQk0 ++ return 0 + local client_container=psmdb-client-66f577db5f-xt92d + kubectl_bin exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.rpa5A4ybkB ++ mktemp + local LAST_ERR=/tmp/tmp.T7Yhx5D26K + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-xt92d -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-21488.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rpa5A4ybkB + cat /tmp/tmp.T7Yhx5D26K + rm /tmp/tmp.rpa5A4ybkB /tmp/tmp.T7Yhx5D26K + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/compare/find.json /tmp/tmp.Xzh2TBRSnh/find + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + desc 'run minio backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-minio ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + desc 'run backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Y8IHd1u9fH ++ mktemp + local LAST_ERR=/tmp/tmp.6dYMlUtIYa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y8IHd1u9fH perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.6dYMlUtIYa + rm /tmp/tmp.Y8IHd1u9fH /tmp/tmp.6dYMlUtIYa + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state................. + '[' -z '' ']' + desc 'run minio backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-gcp-cs ----------------------------------------------------------------------------------- + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + desc 'run backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1987/e2e-tests/data-at-rest-encryption/conf/backup-gcp-cs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.H3apvWk8gd ++ mktemp + local LAST_ERR=/tmp/tmp.aG66bzqRB7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H3apvWk8gd perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.aG66bzqRB7 + rm /tmp/tmp.H3apvWk8gd /tmp/tmp.aG66bzqRB7 + return 0 + wait_backup backup-gcp-cs + local backup_name=backup-gcp-cs + local target_state=ready + set +o xtrace waiting for backup-gcp-cs to reach ready state...2025-07-01T08:48:39.405Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "74f60104-3162-44ac-a96c-dd2b73998faa", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2025-07-01T08:48:45.726Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "848c5c11-eb1a-4053-a8f7-2713088ef7f2", "replset": "cfg", "size": 3, "pods": 2} 2025-07-01T08:48:45.726Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "848c5c11-eb1a-4053-a8f7-2713088ef7f2", "replset": "cfg", "status": "initializing"} 2025-07-01T08:48:45.777Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "848c5c11-eb1a-4053-a8f7-2713088ef7f2", "replset": "rs0", "size": 3, "pods": 2} 2025-07-01T08:48:45.777Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "848c5c11-eb1a-4053-a8f7-2713088ef7f2", "replset": "rs0", "status": "initializing"} 2025-07-01T08:48:46.268Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "848c5c11-eb1a-4053-a8f7-2713088ef7f2", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2025-07-01T08:48:53.557Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "fcf597ab-676b-49c3-88c9-7bb66717bb74", "replset": "cfg", "size": 3, "pods": 2} 2025-07-01T08:48:53.557Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "fcf597ab-676b-49c3-88c9-7bb66717bb74", "replset": "cfg", "status": "initializing"} 2025-07-01T08:48:53.611Z INFO Waiting for the pods {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "fcf597ab-676b-49c3-88c9-7bb66717bb74", "replset": "rs0", "size": 3, "pods": 2} 2025-07-01T08:48:53.611Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "fcf597ab-676b-49c3-88c9-7bb66717bb74", "replset": "rs0", "status": "initializing"} 2025-07-01T08:48:54.078Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "fcf597ab-676b-49c3-88c9-7bb66717bb74", "upgradeInProgress": false, "replsetsReady": 0, "clusterState": "initializing"} 2025-07-01T08:49:10.377Z INFO initiating replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "cfg", "pod": "some-name-cfg-0"} 2025-07-01T08:49:11.909Z INFO replset initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "cfg", "pod": "some-name-cfg-0"} 2025-07-01T08:49:16.910Z INFO creating user admin {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "cfg", "pod": "some-name-cfg-0", "user": "userAdmin"} 2025-07-01T08:49:17.734Z INFO user admin created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "cfg", "pod": "some-name-cfg-0", "user": "userAdmin"} 2025-07-01T08:49:17.893Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "clusterAdmin"} 2025-07-01T08:49:17.949Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "clusterMonitor"} 2025-07-01T08:49:18.001Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "backup"} 2025-07-01T08:49:18.056Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "databaseAdmin"} 2025-07-01T08:49:18.108Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "cfg", "status": "initializing"} 2025-07-01T08:49:28.211Z INFO initiating replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "rs0", "pod": "some-name-rs0-0"} 2025-07-01T08:49:31.953Z INFO replset initialized {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "rs0", "pod": "some-name-rs0-0"} 2025-07-01T08:49:36.953Z INFO creating user admin {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2025-07-01T08:49:39.231Z INFO user admin created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "rs0", "pod": "some-name-rs0-0", "user": "userAdmin"} 2025-07-01T08:49:39.400Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "clusterAdmin"} 2025-07-01T08:49:39.542Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "clusterMonitor"} 2025-07-01T08:49:39.675Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "backup"} 2025-07-01T08:49:39.768Z INFO Creating user {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "database": "admin", "user": "databaseAdmin"} 2025-07-01T08:49:39.847Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "replset": "rs0", "status": "initializing"} 2025-07-01T08:49:40.121Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "name": "some-name-mongos", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "generation": 0, "resourceVersion": ""} 2025-07-01T08:49:40.155Z DEBUG Object created {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "name": "some-name-mongos-", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}", "generation": 0, "resourceVersion": ""} 2025-07-01T08:49:40.770Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6d2bc54-c124-4c55-af12-f98ebe41a336", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "initializing"} 2025-07-01T08:49:42.565Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "_id": 1, "host": "some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"} 2025-07-01T08:49:42.565Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "cfg"} 2025-07-01T08:49:42.565Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "cfg": {"_id":"cfg","version":2,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a1078af156839ee7beb9"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:42.578Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "cfg"} 2025-07-01T08:49:42.578Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "cfg": {"_id":"cfg","version":3,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":0}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a1078af156839ee7beb9"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:42.590Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "liveMembers": 1, "pods": 3} 2025-07-01T08:49:42.594Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "cfg", "status": "initializing"} 2025-07-01T08:49:42.991Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "_id": 1, "host": "some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"} 2025-07-01T08:49:42.991Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "rs0"} 2025-07-01T08:49:42.991Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "cfg": {"_id":"rs0","version":2,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a11b181b1f0dcf461c82"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:43.005Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "rs0"} 2025-07-01T08:49:43.005Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "cfg": {"_id":"rs0","version":3,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":0}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a11b181b1f0dcf461c82"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:43.015Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "liveMembers": 1, "pods": 3} 2025-07-01T08:49:43.016Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "replset": "rs0", "status": "initializing"} 2025-07-01T08:49:43.633Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "10bf2219-1799-45f8-a966-77b5e75019fc", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "initializing"} 2025-07-01T08:49:45.132Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "_id": 2, "host": "some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"} 2025-07-01T08:49:45.132Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "cfg"} 2025-07-01T08:49:45.132Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "cfg": {"_id":"cfg","version":5,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":0},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a1078af156839ee7beb9"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:45.152Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "cfg"} 2025-07-01T08:49:45.152Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "cfg": {"_id":"cfg","version":6,"members":[{"_id":0,"host":"some-name-cfg-0.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-cfg-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-cfg-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-cfg-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"configsvr":true,"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a1078af156839ee7beb9"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:45.180Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "liveMembers": 2, "pods": 3} 2025-07-01T08:49:45.181Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "cfg", "status": "initializing"} 2025-07-01T08:49:45.648Z INFO Adding new member to replset {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "_id": 2, "host": "some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"} 2025-07-01T08:49:45.648Z INFO Adding new nodes {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "rs0"} 2025-07-01T08:49:45.648Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "cfg": {"_id":"rs0","version":5,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":0,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":0},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a11b181b1f0dcf461c82"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:45.662Z INFO Configuring member votes and priorities {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "rs0"} 2025-07-01T08:49:45.663Z DEBUG Running replSetReconfig config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "cfg": {"_id":"rs0","version":6,"members":[{"_id":0,"host":"some-name-rs0-0.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-n9g8","podName":"some-name-rs0-0","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":1,"host":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-gpr4","podName":"some-name-rs0-1","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"secondaryDelaySecs":0,"votes":1},{"_id":2,"host":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":2,"tags":{"nodeName":"gke-jen-psmdb-1987-3dea3-default-pool-60a88c2e-9mfw","podName":"some-name-rs0-2","region":"us-central1","serviceName":"some-name","zone":"us-central1-a"},"votes":1}],"protocolVersion":1,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"getLastErrorDefaults":{"w":1,"wtimeout":0},"replicaSetId":"6863a11b181b1f0dcf461c82"},"writeConcernMajorityJournalDefault":true}} 2025-07-01T08:49:45.679Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "liveMembers": 2, "pods": 3} 2025-07-01T08:49:45.680Z DEBUG Replset status is not healthy {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "replset": "rs0", "status": "initializing"} 2025-07-01T08:49:46.317Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "b6497a7f-1bb4-4d86-bf9c-0e3acdf59e5a", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "initializing"} 2025-07-01T08:49:48.721Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "78b3e160-9104-407a-9a6c-ad67472f004c", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:49:51.224Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "cadb9d79-fea2-44c6-8040-68095e8f1019", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:49:54.849Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "642f47a5-d26f-4a2b-a48f-32704c6a5f8d", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:49:57.550Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "4aeab464-526a-4800-afb2-d998e0c56982", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:49:59.976Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "f6d90123-1a7f-4ba1-a53d-63248dbd6f14", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:50:02.430Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "9dd97b96-c19e-45f3-a4f7-79d4aede9187", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:50:08.357Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "655c6e63-9b10-4bee-b424-bf126f375f14", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:50:10.829Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "480698e3-9852-4359-a226-159204ce1464", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:50:15.976Z DEBUG Cluster is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "ff1f3bb7-cf30-49dd-9c8b-3926ab6d94ca", "upgradeInProgress": false, "replsetsReady": 2, "clusterState": "ready"} 2025-07-01T08:50:24.686Z INFO Cluster state changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "33bc9cfc-6263-4c83-b5b3-86cf687b54ac", "previous": "initializing", "current": "ready"} 2025-07-01T08:50:25.689Z INFO update Mongo version to 7.0.18-11 (fetched from db) {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8"} 2025-07-01T08:50:26.976Z INFO adding rs to shard {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "rs": "rs0"} 2025-07-01T08:50:27.239Z INFO added to shard {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "rs": "rs0"} 2025-07-01T08:50:27.877Z INFO PBM configuration changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "oldHash": "", "newHash": "0a44ff5aea9a7d9fe2e0a315ee8179146004b566af4a39b1e532a5207bad09b8"} 2025-07-01T08:50:28.029Z INFO PBM Setting config {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "cluster": "some-name", "mainStorage": "aws-s3"} 2025-07-01T08:50:28.093Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "name": "azure-blob", "type": "azure"} 2025-07-01T08:50:28.109Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "name": "gcp-cs", "type": "s3"} 2025-07-01T08:50:28.127Z INFO PBM Adding profile {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "name": "minio", "type": "s3"} 2025-07-01T08:50:28.133Z INFO PBM resync storage {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "storage": "aws-s3"} 2025-07-01T08:50:28.133Z DEBUG PBM main storage changed {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "some-name", "reconcileID": "075bf91a-8bd2-4ce2-b478-6b1141d3fde8", "old": {"type":""}, "new": {"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","bucket":"operator-testing","prefix":"psmdb","insecureSkipTLSVerify":false}}} 2025-07-01T08:50:45.149Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e"} 2025-07-01T08:50:45.161Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "state": ""} 2025-07-01T08:50:45.445Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:45.445Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:45.530Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e"} 2025-07-01T08:50:45.680Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "finalizer": "internal.percona.com/release-lock"} 2025-07-01T08:50:45.680Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "backup": "backup-minio", "storage": "minio"} 2025-07-01T08:50:45.680Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "backup": "backup-minio", "storage": "minio", "backupCmd": "backup [name: 2025-07-01T08:50:45Z, compression: gzip (level: default)] ", "profile": "minio"} 2025-07-01T08:50:45.687Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e", "previous": "", "current": "requested"} 2025-07-01T08:50:45.721Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "e49dfa51-94cd-4907-a264-7a18e2084d0e"} 2025-07-01T08:50:45.721Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003"} 2025-07-01T08:50:45.739Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003", "state": "requested"} 2025-07-01T08:50:45.813Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:45.813Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:45.893Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003"} 2025-07-01T08:50:45.895Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003", "pbmName": "2025-07-01T08:50:45Z", "backup": "backup-minio"} 2025-07-01T08:50:45.896Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "f8fbd005-3bca-4b6b-9bd9-5818fbb65003"} 2025-07-01T08:50:45.896Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40"} 2025-07-01T08:50:45.907Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40", "state": "requested"} 2025-07-01T08:50:45.992Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:45.992Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:46.073Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40"} 2025-07-01T08:50:46.075Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359845,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359845,"I":18},"status":"starting","conditions":[{"timestamp":1751359845,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:50:46.077Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "89091e1d-8145-4135-bef7-d90d392f4b40"} 2025-07-01T08:50:50.722Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2"} 2025-07-01T08:50:50.734Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2", "state": "requested"} 2025-07-01T08:50:50.810Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:50.810Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:50.885Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2"} 2025-07-01T08:50:50.887Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"dumpDone","last_transition_ts":1751359850,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"dumpDone","iscs":true,"last_transition_ts":1751359849,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359846,"I":12},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:50:50.889Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2", "previous": "requested", "current": "running"} 2025-07-01T08:50:50.911Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d28b38aa-5ff6-423b-b10e-754319df70f2"} 2025-07-01T08:50:50.913Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508"} 2025-07-01T08:50:50.930Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508", "state": "running"} 2025-07-01T08:50:51.005Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:51.005Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:51.078Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508"} 2025-07-01T08:50:51.079Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"dumpDone","last_transition_ts":1751359850,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"dumpDone","iscs":true,"last_transition_ts":1751359849,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359846,"I":12},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:50:51.083Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d1dcd8fa-3ee3-4b79-9211-f842a9cc2508"} 2025-07-01T08:50:55.912Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8"} 2025-07-01T08:50:55.928Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8", "state": "running"} 2025-07-01T08:50:56.005Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:50:56.005Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:50:56.075Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8"} 2025-07-01T08:50:56.077Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"dumpDone","last_transition_ts":1751359850,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"dumpDone","iscs":true,"last_transition_ts":1751359849,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359851,"I":5},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:50:56.079Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "12bfcd7d-54ab-4615-97f4-51daae67eba8"} 2025-07-01T08:51:01.080Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96"} 2025-07-01T08:51:01.093Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96", "state": "running"} 2025-07-01T08:51:01.162Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:01.162Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:51:01.239Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96"} 2025-07-01T08:51:01.240Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"dumpDone","last_transition_ts":1751359850,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"dumpDone","iscs":true,"last_transition_ts":1751359849,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359856,"I":2},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:51:01.243Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "b7656bae-9d0f-40d9-8190-dc232757cc96"} 2025-07-01T08:51:06.244Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb"} 2025-07-01T08:51:06.258Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb", "state": "running"} 2025-07-01T08:51:06.350Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:06.351Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:51:06.445Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb"} 2025-07-01T08:51:06.447Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"done","last_transition_ts":1751359862,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":1751359849,"I":67},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"},{"timestamp":1751359862,"status":"done"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"dumpDone","iscs":true,"last_transition_ts":1751359849,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":12007,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359866,"I":1},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:51:06.451Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "ac3f7ae0-08f4-4341-88d9-78027eb08eeb"} 2025-07-01T08:51:11.451Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991"} 2025-07-01T08:51:11.465Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991", "state": "running"} 2025-07-01T08:51:11.557Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:11.557Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:51:11.634Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991"} 2025-07-01T08:51:11.636Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"done","last_transition_ts":1751359862,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":1751359849,"I":67},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"},{"timestamp":1751359862,"status":"done"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"done","iscs":true,"last_transition_ts":1751359870,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"},{"timestamp":1751359870,"status":"done"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":94093,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359850,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359849,"I":67},"hb":{"T":1751359871,"I":2},"status":"dumpDone","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:51:11.639Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "beb1ee0c-6ffb-45f2-87b9-2f9eb8fc1991"} 2025-07-01T08:51:16.639Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0"} 2025-07-01T08:51:16.654Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "state": "running"} 2025-07-01T08:51:16.733Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:16.733Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "currentJob": {"Name":"backup-minio","Type":0}} 2025-07-01T08:51:16.807Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0"} 2025-07-01T08:51:16.809Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "backup": "backup-minio", "pbmName": "2025-07-01T08:50:45Z", "meta": {"type":"logical","opid":"6863a165dfe625e6ace2b8ab","name":"2025-07-01T08:50:45Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-07-01T08:50:45Z/rs0/metadata.json","oplog_name":"2025-07-01T08:50:45Z/rs0/oplog","start_ts":1751359846,"status":"done","last_transition_ts":1751359862,"first_write_ts":{"T":1751359840,"I":13},"last_write_ts":{"T":1751359849,"I":67},"node":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"},{"timestamp":1751359862,"status":"done"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-07-01T08:50:45Z/cfg/metadata.json","oplog_name":"2025-07-01T08:50:45Z/cfg/oplog","start_ts":1751359846,"status":"done","iscs":true,"last_transition_ts":1751359870,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359850,"I":12},"node":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","conditions":[{"timestamp":1751359846,"status":"running"},{"timestamp":1751359849,"status":"dumpDone"},{"timestamp":1751359870,"status":"done"}],"pbm_version":"2.10.0","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":94093,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359845,"last_transition_ts":1751359871,"first_write_ts":{"T":1751359846,"I":10},"last_write_ts":{"T":1751359849,"I":67},"hb":{"T":1751359871,"I":2},"status":"done","conditions":[{"timestamp":1751359845,"status":"starting"},{"timestamp":1751359849,"status":"running"},{"timestamp":1751359850,"status":"dumpDone"},{"timestamp":1751359871,"status":"done"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:51:16.811Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "previous": "running", "current": "ready"} 2025-07-01T08:51:16.833Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0", "lease": "psmdb-some-name-backup-lock"} 2025-07-01T08:51:16.861Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "6296f647-d73e-4852-8b91-d07965f7f8b0"} 2025-07-01T08:51:16.862Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "9d519bc7-2f03-4d43-884e-ee239df96edd"} 2025-07-01T08:51:16.875Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "9d519bc7-2f03-4d43-884e-ee239df96edd", "state": "ready"} 2025-07-01T08:51:16.875Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "9d519bc7-2f03-4d43-884e-ee239df96edd"} 2025-07-01T08:51:20.339Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15"} 2025-07-01T08:51:20.358Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "state": ""} 2025-07-01T08:51:20.439Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:20.439Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-07-01T08:51:20.514Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15"} 2025-07-01T08:51:20.548Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "finalizer": "internal.percona.com/release-lock"} 2025-07-01T08:51:20.548Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "backup": "backup-gcp-cs", "storage": "gcp-cs"} 2025-07-01T08:51:20.548Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "backup": "backup-gcp-cs", "storage": "gcp-cs", "backupCmd": "backup [name: 2025-07-01T08:51:20Z, compression: gzip (level: default)] ", "profile": "gcp-cs"} 2025-07-01T08:51:20.555Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15", "previous": "", "current": "requested"} 2025-07-01T08:51:20.582Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "78c38c5b-a925-4704-9225-de13b9ba3c15"} 2025-07-01T08:51:20.583Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506"} 2025-07-01T08:51:20.602Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506", "state": "requested"} 2025-07-01T08:51:20.688Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:20.688Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-07-01T08:51:20.762Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506"} 2025-07-01T08:51:20.764Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506", "pbmName": "2025-07-01T08:51:20Z", "backup": "backup-gcp-cs"} 2025-07-01T08:51:20.766Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "b50d2323-2ed8-4feb-99b7-e8751016a506"} 2025-07-01T08:51:20.766Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb"} 2025-07-01T08:51:20.780Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb", "state": "requested"} 2025-07-01T08:51:20.854Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:20.854Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-07-01T08:51:20.926Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb"} 2025-07-01T08:51:20.927Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb", "pbmName": "2025-07-01T08:51:20Z", "backup": "backup-gcp-cs"} 2025-07-01T08:51:20.929Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "7f6f99f9-e0f5-480b-8b16-1648618d87eb"} 2025-07-01T08:51:21.862Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d095a652-879f-46b8-a2cd-9e299ae9b3a3"} 2025-07-01T08:51:21.885Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d095a652-879f-46b8-a2cd-9e299ae9b3a3", "state": "ready"} 2025-07-01T08:51:21.885Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-minio", "reconcileID": "d095a652-879f-46b8-a2cd-9e299ae9b3a3"} 2025-07-01T08:51:25.583Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b"} 2025-07-01T08:51:25.595Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "state": "requested"} 2025-07-01T08:51:25.670Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "cluster": "some-name", "namespace": "data-at-rest-encryption-21488"} 2025-07-01T08:51:25.670Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-07-01T08:51:25.745Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b"} 2025-07-01T08:51:25.748Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "backup": "backup-gcp-cs", "pbmName": "2025-07-01T08:51:20Z", "meta": {"type":"logical","opid":"6863a188dfe625e6ace2b8f8","name":"2025-07-01T08:51:20Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1751359880,"last_transition_ts":1751359881,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1751359881,"I":22},"status":"error","conditions":[{"timestamp":1751359880,"status":"starting"},{"timestamp":1751359881,"status":"error","error":"check read access: file stat: get S3 object header: operation error S3: HeadObject, https response error StatusCode: 403, RequestID: , HostID: , api error Forbidden: Forbidden"}],"n":[{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-21488.svc.cluster.local:27017"},{"rs":"cfg","n":["some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017","some-name-cfg-1.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"],"ack":"some-name-cfg-2.some-name-cfg.data-at-rest-encryption-21488.svc.cluster.local:27017"}],"error":"check read access: file stat: get S3 object header: operation error S3: HeadObject, https response error StatusCode: 403, RequestID: , HostID: , api error Forbidden: Forbidden","pbm_version":"2.10.0","balancer":"full"}} 2025-07-01T08:51:25.750Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "previous": "requested", "current": "error"} 2025-07-01T08:51:25.771Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b", "lease": "psmdb-some-name-backup-lock"} 2025-07-01T08:51:25.794Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "077f97e7-b1d5-4a16-98cf-59b8ee63e31b"} 2025-07-01T08:51:25.794Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "88adb804-14c1-4c1e-9a87-6fd3c0ba5fc0"} 2025-07-01T08:51:25.807Z DEBUG Got object from API server {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "88adb804-14c1-4c1e-9a87-6fd3c0ba5fc0", "state": "error"} 2025-07-01T08:51:25.807Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-21488"}, "namespace": "data-at-rest-encryption-21488", "name": "backup-gcp-cs", "reconcileID": "88adb804-14c1-4c1e-9a87-6fd3c0ba5fc0"} NAME CLUSTER STORAGE DESTINATION TYPE SIZE STATUS COMPLETED AGE backup-gcp-cs some-name gcp-cs s3://operator-testing/psmdb/2025-07-01T08:51:20Z logical error 8s backup-minio some-name minio s3://operator-testing/2025-07-01T08:50:45Z logical 91.89KB ready 17s 43s Name: backup-gcp-cs Namespace: data-at-rest-encryption-21488 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBBackup Metadata: Creation Timestamp: 2025-07-01T08:51:20Z Finalizers: internal.percona.com/release-lock Generation: 1 Resource Version: 1751359885763407017 UID: 079a363f-932b-484f-8985-f175e8e88528 Spec: Cluster Name: some-name Storage Name: gcp-cs Type: logical Status: Destination: s3://operator-testing/psmdb/2025-07-01T08:51:20Z Error: check read access: file stat: get S3 object header: operation error S3: HeadObject, https response error StatusCode: 403, RequestID: , HostID: , api error Forbidden: Forbidden Last Transition: 2025-07-01T08:51:21Z Pbm Name: 2025-07-01T08:51:20Z Replset Names: cfg rs0 s3: Bucket: operator-testing Credentials Secret: gcp-cs-secret Endpoint URL: https://storage.googleapis.com Prefix: psmdb Region: us-east-1 Server Side Encryption: Start: 2025-07-01T08:51:20Z State: error Storage Name: gcp-cs Type: logical Events: Backup object psmdb-backup/backup-gcp-cs is in error state.