Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/logs/data-at-rest-encryption.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + create_infra data-at-rest-encryption-32271 + local ns=data-at-rest-encryption-32271 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.YjXSenpmhJ ++ mktemp + local LAST_ERR=/tmp/tmp.n2LwJ57zSg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YjXSenpmhJ customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.n2LwJ57zSg + rm /tmp/tmp.YjXSenpmhJ /tmp/tmp.n2LwJ57zSg + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.lFW61XO9YH ++ mktemp + local LAST_ERR=/tmp/tmp.ABlq9q6DcX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lFW61XO9YH customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.ABlq9q6DcX + rm /tmp/tmp.lFW61XO9YH /tmp/tmp.ABlq9q6DcX + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.YSyCtdAKPE ++ mktemp + local LAST_ERR=/tmp/tmp.MjBV5L29jV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YSyCtdAKPE + cat /tmp/tmp.MjBV5L29jV + rm /tmp/tmp.YSyCtdAKPE /tmp/tmp.MjBV5L29jV + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ERP8IYD6a6 ++ mktemp + local LAST_ERR=/tmp/tmp.LpMs77646U + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ERP8IYD6a6 + cat /tmp/tmp.LpMs77646U + rm /tmp/tmp.ERP8IYD6a6 /tmp/tmp.LpMs77646U + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.TaBiJ7WTRT ++ mktemp + local LAST_ERR=/tmp/tmp.Mi1HtnDpNK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TaBiJ7WTRT clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.Mi1HtnDpNK + rm /tmp/tmp.TaBiJ7WTRT /tmp/tmp.Mi1HtnDpNK + return 0 + check_crd_for_deletion PR-1926-2614d852 + local git_tag=PR-1926-2614d852 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1926-2614d852/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.23Qzoct0BF +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZTd8TKQ6de ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.23Qzoct0BF ++ cat /tmp/tmp.ZTd8TKQ6de Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.23Qzoct0BF ++ cat /tmp/tmp.ZTd8TKQ6de Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.23Qzoct0BF ++ cat /tmp/tmp.ZTd8TKQ6de Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.23Qzoct0BF ++ cat /tmp/tmp.ZTd8TKQ6de Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.23Qzoct0BF /tmp/tmp.ZTd8TKQ6de ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + awk '{print$1}' + local LAST_OUT=/tmp/tmp.CcnY6LIZX2 ++ mktemp + local LAST_OUT=/tmp/tmp.v4RHWNkFDi ++ mktemp + local LAST_ERR=/tmp/tmp.MWd37Gygcj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found ++ mktemp + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + local LAST_ERR=/tmp/tmp.GeeawTBpsW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + xargs kubectl delete ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CcnY6LIZX2 + cat /tmp/tmp.GeeawTBpsW + rm /tmp/tmp.CcnY6LIZX2 /tmp/tmp.GeeawTBpsW + return 0 namespace "data-at-rest-encryption-23392" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v4RHWNkFDi namespace "psmdb-operator" deleted + cat /tmp/tmp.MWd37Gygcj + rm /tmp/tmp.v4RHWNkFDi /tmp/tmp.MWd37Gygcj + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.LBfgHWHs7o ++ mktemp + local LAST_ERR=/tmp/tmp.z2AS7jVJXL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LBfgHWHs7o + cat /tmp/tmp.z2AS7jVJXL + rm /tmp/tmp.LBfgHWHs7o /tmp/tmp.z2AS7jVJXL + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.d1ydwtBubB ++ mktemp + local LAST_ERR=/tmp/tmp.EB4it1KqSg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.d1ydwtBubB namespace/psmdb-operator created + cat /tmp/tmp.EB4it1KqSg + rm /tmp/tmp.d1ydwtBubB /tmp/tmp.EB4it1KqSg + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.1vRTVBX4I8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.35VFBlVw4J ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1vRTVBX4I8 ++ cat /tmp/tmp.35VFBlVw4J ++ rm /tmp/tmp.1vRTVBX4I8 /tmp/tmp.35VFBlVw4J ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.32fkkou45j ++ mktemp + local LAST_ERR=/tmp/tmp.VpFb9rsCsc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.32fkkou45j Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8" modified. + cat /tmp/tmp.VpFb9rsCsc + rm /tmp/tmp.32fkkou45j /tmp/tmp.VpFb9rsCsc + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.jsna0QlaoR ++ mktemp + local LAST_ERR=/tmp/tmp.D7xJCfJ2aq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jsna0QlaoR customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.D7xJCfJ2aq + rm /tmp/tmp.jsna0QlaoR /tmp/tmp.D7xJCfJ2aq + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + kubectl_bin apply -n psmdb-operator -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml + local LAST_OUT=/tmp/tmp.SJ2JlUdTtT ++ mktemp + local LAST_ERR=/tmp/tmp.G8oVPo2HNq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SJ2JlUdTtT clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.G8oVPo2HNq + rm /tmp/tmp.SJ2JlUdTtT /tmp/tmp.G8oVPo2HNq + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1926-2614d852") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.5Fvejn65ra ++ mktemp + local LAST_ERR=/tmp/tmp.aM5RtQIE35 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5Fvejn65ra deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.aM5RtQIE35 + rm /tmp/tmp.5Fvejn65ra /tmp/tmp.aM5RtQIE35 + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.V801pOyYad +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y3C2VjZJle ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V801pOyYad ++ cat /tmp/tmp.Y3C2VjZJle ++ rm /tmp/tmp.V801pOyYad /tmp/tmp.Y3C2VjZJle ++ return 0 + wait_pod percona-server-mongodb-operator-87cfc696-m4gdn + local pod=percona-server-mongodb-operator-87cfc696-m4gdn + set +o xtrace waiting for pod/percona-server-mongodb-operator-87cfc696-m4gdn to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.bay3aO694P +++ mktemp ++ local LAST_ERR=/tmp/tmp.AIn48cQROV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bay3aO694P ++ cat /tmp/tmp.AIn48cQROV ++ rm /tmp/tmp.bay3aO694P /tmp/tmp.AIn48cQROV ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-87cfc696-m4gdn ++ mktemp + local LAST_OUT=/tmp/tmp.3CCon0o2Dx ++ mktemp + local LAST_ERR=/tmp/tmp.63v71nICJI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs percona-server-mongodb-operator-87cfc696-m4gdn + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3CCon0o2Dx + cat /tmp/tmp.63v71nICJI + rm /tmp/tmp.3CCon0o2Dx /tmp/tmp.63v71nICJI + return 0 2025-05-21T22:23:41.904Z INFO setup Manager starting up {"gitCommit": "2614d8526fb6cb612447e95aa985aca9a64a3da0", "gitBranch": "PR-1926-2614d852", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"} + create_namespace data-at-rest-encryption-32271 + local namespace=data-at-rest-encryption-32271 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces data-at-rest-encryption-32271' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces data-at-rest-encryption-32271 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace data-at-rest-encryption-32271 --ignore-not-found + xargs kubectl delete ns + awk '{print$1}' + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.LiJzZF4Eqi ++ mktemp + local LAST_ERR=/tmp/tmp.UpuSluHuF5 + local exit_status=0 + local timeout=4 + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.FMHJrfvobb ++ mktemp + local LAST_ERR=/tmp/tmp.DfNPNlR4N0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace data-at-rest-encryption-32271 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LiJzZF4Eqi + cat /tmp/tmp.UpuSluHuF5 + rm /tmp/tmp.LiJzZF4Eqi /tmp/tmp.UpuSluHuF5 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FMHJrfvobb + cat /tmp/tmp.DfNPNlR4N0 + rm /tmp/tmp.FMHJrfvobb /tmp/tmp.DfNPNlR4N0 + return 0 + kubectl_bin wait --for=delete namespace data-at-rest-encryption-32271 ++ mktemp + local LAST_OUT=/tmp/tmp.MQxq7hM9uL ++ mktemp + local LAST_ERR=/tmp/tmp.GIHn7eZHH7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace data-at-rest-encryption-32271 namespace "gke-managed-cim" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MQxq7hM9uL + cat /tmp/tmp.GIHn7eZHH7 + rm /tmp/tmp.MQxq7hM9uL /tmp/tmp.GIHn7eZHH7 + return 0 + desc 'create namespace data-at-rest-encryption-32271' + set +o xtrace ----------------------------------------------------------------------------------- create namespace data-at-rest-encryption-32271 ----------------------------------------------------------------------------------- + kubectl_bin create namespace data-at-rest-encryption-32271 ++ mktemp + local LAST_OUT=/tmp/tmp.H1ZhkLyd3r ++ mktemp + local LAST_ERR=/tmp/tmp.6KtzFPlB7S + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace data-at-rest-encryption-32271 namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H1ZhkLyd3r namespace/data-at-rest-encryption-32271 created + cat /tmp/tmp.6KtzFPlB7S + rm /tmp/tmp.H1ZhkLyd3r /tmp/tmp.6KtzFPlB7S + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.riRFj6wPmT +++ mktemp ++ local LAST_ERR=/tmp/tmp.0N5G5eZJKn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.riRFj6wPmT ++ cat /tmp/tmp.0N5G5eZJKn ++ rm /tmp/tmp.riRFj6wPmT /tmp/tmp.0N5G5eZJKn ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8 --namespace=data-at-rest-encryption-32271 ++ mktemp + local LAST_OUT=/tmp/tmp.gGK8JG5Z2M ++ mktemp + local LAST_ERR=/tmp/tmp.BrekWhyRS3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8 --namespace=data-at-rest-encryption-32271 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gGK8JG5Z2M Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-2614d852-11-cluster8" modified. + cat /tmp/tmp.BrekWhyRS3 + rm /tmp/tmp.gGK8JG5Z2M /tmp/tmp.BrekWhyRS3 + return 0 + deploy_vault + name=vault-service + desc 'install Vault vault-service' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service ----------------------------------------------------------------------------------- + helm uninstall vault-service Error: uninstall: Release not loaded: vault-service: release: not found + : + helm repo remove hashicorp "hashicorp" has been removed from your repositories + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" has been added to your repositories + destroy_vault ++ helm list --all-namespaces --filter vault-service ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local vault_ns= + desc 'destroy vault' + set +o xtrace ----------------------------------------------------------------------------------- destroy vault ----------------------------------------------------------------------------------- ++ grep vault ++ kubectl api-resources ++ awk '{print $1}' + '[' -n '' ']' ++ kubectl get crd ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted ++ kubectl get clusterrole ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted ++ kubectl get mutatingwebhookconfiguration ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted + [[ -n '' ]] + retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false + local max=10 + local delay=60 + shift 2 + local n=1 + helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false NAME: vault-service LAST DEPLOYED: Wed May 21 22:24:12 2025 NAMESPACE: data-at-rest-encryption-32271 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service. To learn more about the release, try: $ helm status vault-service $ helm get manifest vault-service + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' Running + kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.XiRdED9cu5 ++ mktemp + local LAST_ERR=/tmp/tmp.kV9hZ6VdpF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XiRdED9cu5 + cat /tmp/tmp.kV9hZ6VdpF + rm /tmp/tmp.XiRdED9cu5 /tmp/tmp.kV9hZ6VdpF + return 0 ++ jq -r '.unseal_keys_b64[]' + local unsealKey=7XmrcNhQ78ruTPJqVYpVVCYGp44pEbQ1pbb8qP5ru5E= ++ jq -r .root_token + local token=hvs.yx8oyIytMm1Binnq0N56j4hj + kubectl_bin exec pod/vault-service-0 -- vault operator unseal 7XmrcNhQ78ruTPJqVYpVVCYGp44pEbQ1pbb8qP5ru5E= ++ mktemp + local LAST_OUT=/tmp/tmp.t7AZfCT7Ub ++ mktemp + local LAST_ERR=/tmp/tmp.HhrPQuu43f + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator unseal 7XmrcNhQ78ruTPJqVYpVVCYGp44pEbQ1pbb8qP5ru5E= + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t7AZfCT7Ub Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 1.19.0 Build Date 2025-03-04T12:36:40Z Storage Type file Cluster Name vault-cluster-9958e124 Cluster ID 60be22af-176a-d50e-60b8-5c2d64c47177 HA Enabled false + cat /tmp/tmp.HhrPQuu43f + rm /tmp/tmp.t7AZfCT7Ub /tmp/tmp.HhrPQuu43f + return 0 + kubectl_bin exec -it pod/vault-service-0 -- sh ++ mktemp + local LAST_OUT=/tmp/tmp.Y5RgyD8wk9 ++ mktemp + local LAST_ERR=/tmp/tmp.P38VtyZr3M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it pod/vault-service-0 -- sh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y5RgyD8wk9 Success! You are now authenticated. The token information displayed below is already stored in the token helper. You do NOT need to run "vault login" again. Future Vault requests will automatically use this token. Key Value --- ----- token hvs.yx8oyIytMm1Binnq0N56j4hj token_accessor 60ggkqDBUmuNEdiLaqvrtngY token_duration ∞ token_renewable false token_policies ["root"] identity_policies [] policies ["root"] Success! Enabled the kv-v2 secrets engine at: secret/ + cat /tmp/tmp.P38VtyZr3M Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.Y5RgyD8wk9 /tmp/tmp.P38VtyZr3M + return 0 + kubectl_bin create secret generic vault-secret --from-literal=token=hvs.yx8oyIytMm1Binnq0N56j4hj ++ mktemp + local LAST_OUT=/tmp/tmp.lCBTdZRhiV ++ mktemp + local LAST_ERR=/tmp/tmp.JuH2RhpxbU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create secret generic vault-secret --from-literal=token=hvs.yx8oyIytMm1Binnq0N56j4hj + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lCBTdZRhiV secret/vault-secret created + cat /tmp/tmp.JuH2RhpxbU + rm /tmp/tmp.lCBTdZRhiV /tmp/tmp.JuH2RhpxbU + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Wed May 21 22:24:48 2025 NAMESPACE: data-at-rest-encryption-32271 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.data-at-rest-encryption-32271.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace data-at-rest-encryption-32271 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace data-at-rest-encryption-32271 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace data-at-rest-encryption-32271 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace data-at-rest-encryption-32271 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9LHor8JVdR +++ mktemp ++ local LAST_ERR=/tmp/tmp.bDFTkY76uR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9LHor8JVdR ++ cat /tmp/tmp.bDFTkY76uR ++ rm /tmp/tmp.9LHor8JVdR /tmp/tmp.bDFTkY76uR ++ return 0 + MINIO_POD=minio-service-86dfccd949-zt7zs + wait_pod minio-service-86dfccd949-zt7zs + local pod=minio-service-86dfccd949-zt7zs + set +o xtrace waiting for pod/minio-service-86dfccd949-zt7zs to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-32271.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.ZXh0ToYVeR ++ mktemp + local LAST_ERR=/tmp/tmp.zXOAVUWlId + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-32271.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZXh0ToYVeR service/minio-service created + cat /tmp/tmp.zXOAVUWlId + rm /tmp/tmp.ZXh0ToYVeR /tmp/tmp.zXOAVUWlId + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.dIvdccu3EW ++ mktemp + local LAST_ERR=/tmp/tmp.bEPg0YugB7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dIvdccu3EW pod "aws-cli" deleted + cat /tmp/tmp.bEPg0YugB7 If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.dIvdccu3EW /tmp/tmp.bEPg0YugB7 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.j2AUKSxPuL ++ mktemp + local LAST_ERR=/tmp/tmp.Ne1l0rcxPu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j2AUKSxPuL secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.Ne1l0rcxPu + rm /tmp/tmp.j2AUKSxPuL /tmp/tmp.Ne1l0rcxPu + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.KQ6CweJCFr ++ mktemp + local LAST_ERR=/tmp/tmp.O2cLl1ZOc8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KQ6CweJCFr secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.O2cLl1ZOc8 + rm /tmp/tmp.KQ6CweJCFr /tmp/tmp.O2cLl1ZOc8 + return 0 + cluster=some-name + desc 'create PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/some-name.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GyM4LJTOOt ++ mktemp + local LAST_ERR=/tmp/tmp.ArmtsZssp4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/some-name.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1926-2614d852"' + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GyM4LJTOOt perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.ArmtsZssp4 + rm /tmp/tmp.GyM4LJTOOt /tmp/tmp.ArmtsZssp4 + return 0 + desc 'check if all Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 false + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready........OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready....OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T7m8vrSytD +++ mktemp ++ local LAST_ERR=/tmp/tmp.dMiOEqPYso ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.T7m8vrSytD ++ cat /tmp/tmp.dMiOEqPYso ++ rm /tmp/tmp.T7m8vrSytD /tmp/tmp.dMiOEqPYso ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fPabK1I8s6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ottjB1OJAT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fPabK1I8s6 ++ cat /tmp/tmp.ottjB1OJAT ++ rm /tmp/tmp.fPabK1I8s6 /tmp/tmp.ottjB1OJAT ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9BBp3f5ygp +++ mktemp ++ local LAST_ERR=/tmp/tmp.u70nI6waF3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9BBp3f5ygp ++ cat /tmp/tmp.u70nI6waF3 ++ rm /tmp/tmp.9BBp3f5ygp /tmp/tmp.u70nI6waF3 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hggOlzIXlZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.nUiuethFma ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hggOlzIXlZ ++ cat /tmp/tmp.nUiuethFma ++ rm /tmp/tmp.hggOlzIXlZ /tmp/tmp.nUiuethFma ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready....OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t0bGJBMCrz +++ mktemp ++ local LAST_ERR=/tmp/tmp.AIvzLaS2FU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.t0bGJBMCrz ++ cat /tmp/tmp.AIvzLaS2FU ++ rm /tmp/tmp.t0bGJBMCrz /tmp/tmp.AIvzLaS2FU ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wvXvJaNCli +++ mktemp ++ local LAST_ERR=/tmp/tmp.mr2IIkwqzE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wvXvJaNCli ++ cat /tmp/tmp.mr2IIkwqzE ++ rm /tmp/tmp.wvXvJaNCli /tmp/tmp.mr2IIkwqzE ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.qEnW8K520S/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-32271", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.zfDGftG4No ++ mktemp + local LAST_ERR=/tmp/tmp.gpBSfgtweh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zfDGftG4No + cat /tmp/tmp.gpBSfgtweh + rm /tmp/tmp.zfDGftG4No /tmp/tmp.gpBSfgtweh + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.qEnW8K520S/statefulset_some-name-rs0.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.qEnW8K520S/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.qEnW8K520S/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml /tmp/tmp.qEnW8K520S/statefulset_some-name-rs0.yml + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.qEnW8K520S/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-32271", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.AaEW9j7557 ++ mktemp + local LAST_ERR=/tmp/tmp.yZp7eZPrK9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AaEW9j7557 + cat /tmp/tmp.yZp7eZPrK9 + rm /tmp/tmp.AaEW9j7557 /tmp/tmp.yZp7eZPrK9 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.qEnW8K520S/statefulset_some-name-cfg.yml + version_gt 1.22 ++ bc -l ++ echo '1.30 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.qEnW8K520S/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.qEnW8K520S/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml /tmp/tmp.qEnW8K520S/statefulset_some-name-cfg.yml + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-32271 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-32271 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.deTAm3RwjQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.yX0XqHINos ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.deTAm3RwjQ ++ cat /tmp/tmp.yX0XqHINos ++ rm /tmp/tmp.deTAm3RwjQ /tmp/tmp.yX0XqHINos ++ return 0 + local client_container=psmdb-client-66f577db5f-7wtwc + kubectl_bin exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.y9lNTDZHM2 ++ mktemp + local LAST_ERR=/tmp/tmp.NWezGGcHj1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.y9lNTDZHM2 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("e1d04c94-fde8-4c75-9555-c0b8a3b1bb38") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.NWezGGcHj1 + rm /tmp/tmp.y9lNTDZHM2 /tmp/tmp.NWezGGcHj1 + return 0 + sleep 2 + desc 'write data, read it' + set +o xtrace ----------------------------------------------------------------------------------- write data, read it ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JTm7INa6mS +++ mktemp ++ local LAST_ERR=/tmp/tmp.qSQGoZE8vF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JTm7INa6mS ++ cat /tmp/tmp.qSQGoZE8vF ++ rm /tmp/tmp.JTm7INa6mS /tmp/tmp.qSQGoZE8vF ++ return 0 + local client_container=psmdb-client-66f577db5f-7wtwc + kubectl_bin exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.kDDnrunSfw ++ mktemp + local LAST_ERR=/tmp/tmp.TGoumwp4Wm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kDDnrunSfw Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("df05ab34-c982-4b5c-aadf-32e5d5e602f1") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.TGoumwp4Wm + rm /tmp/tmp.kDDnrunSfw /tmp/tmp.TGoumwp4Wm + return 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 + local command=find + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-32271 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZFfsV8B8Pa +++ mktemp ++ local LAST_ERR=/tmp/tmp.TAi6feYahq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZFfsV8B8Pa ++ cat /tmp/tmp.TAi6feYahq ++ rm /tmp/tmp.ZFfsV8B8Pa /tmp/tmp.TAi6feYahq ++ return 0 + local client_container=psmdb-client-66f577db5f-7wtwc + kubectl_bin exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.V9imQYQyfW ++ mktemp + local LAST_ERR=/tmp/tmp.KcHRyIPXxu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-7wtwc -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-32271.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V9imQYQyfW + cat /tmp/tmp.KcHRyIPXxu + rm /tmp/tmp.V9imQYQyfW /tmp/tmp.KcHRyIPXxu + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/compare/find.json /tmp/tmp.qEnW8K520S/find + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + desc 'run minio backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-minio ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + desc 'run backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.H7I99XrA2K ++ mktemp + local LAST_ERR=/tmp/tmp.nmJZHcceV8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H7I99XrA2K perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.nmJZHcceV8 + rm /tmp/tmp.H7I99XrA2K /tmp/tmp.nmJZHcceV8 + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state............... + '[' -z '' ']' + desc 'run minio backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-gcp-cs ----------------------------------------------------------------------------------- + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + desc 'run backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/data-at-rest-encryption/conf/backup-gcp-cs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.9isZZ56kTo ++ mktemp + local LAST_ERR=/tmp/tmp.WpNpSESbiC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9isZZ56kTo perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.WpNpSESbiC + rm /tmp/tmp.9isZZ56kTo /tmp/tmp.WpNpSESbiC + return 0 + wait_backup backup-gcp-cs + local backup_name=backup-gcp-cs + local target_state=ready + set +o xtrace waiting for backup-gcp-cs to reach ready state........................................................2025-05-21T22:28:36.240Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "ee62f849-b9c3-47d2-8af1-0dea5c771d5e", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:36.240Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "ee62f849-b9c3-47d2-8af1-0dea5c771d5e", "currentJob": {"Name":"backup-minio","Type":0}} 2025-05-21T22:28:36.320Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "ee62f849-b9c3-47d2-8af1-0dea5c771d5e"} 2025-05-21T22:28:36.322Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "ee62f849-b9c3-47d2-8af1-0dea5c771d5e", "backup": "backup-minio", "pbmName": "2025-05-21T22:28:20Z", "meta": {"type":"logical","opid":"682e53846f39cb2ec07effc8","name":"2025-05-21T22:28:20Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-21T22:28:20Z/rs0/metadata.json","oplog_name":"2025-05-21T22:28:20Z/rs0/oplog","start_ts":1747866501,"status":"dumpDone","last_transition_ts":1747866505,"first_write_ts":{"T":1747866494,"I":12},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-21T22:28:20Z/cfg/metadata.json","oplog_name":"2025-05-21T22:28:20Z/cfg/oplog","start_ts":1747866501,"status":"dumpDone","iscs":true,"last_transition_ts":1747866504,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":21},"node":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866504,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866501,"last_transition_ts":1747866505,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866511,"I":3},"status":"dumpDone","conditions":[{"timestamp":1747866501,"status":"starting"},{"timestamp":1747866504,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:28:36.326Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "ee62f849-b9c3-47d2-8af1-0dea5c771d5e"} 2025-05-21T22:28:41.327Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59"} 2025-05-21T22:28:41.420Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:41.420Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59", "currentJob": {"Name":"backup-minio","Type":0}} 2025-05-21T22:28:41.499Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59"} 2025-05-21T22:28:41.500Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59", "backup": "backup-minio", "pbmName": "2025-05-21T22:28:20Z", "meta": {"type":"logical","opid":"682e53846f39cb2ec07effc8","name":"2025-05-21T22:28:20Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-21T22:28:20Z/rs0/metadata.json","oplog_name":"2025-05-21T22:28:20Z/rs0/oplog","start_ts":1747866501,"status":"dumpDone","last_transition_ts":1747866505,"first_write_ts":{"T":1747866494,"I":12},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-21T22:28:20Z/cfg/metadata.json","oplog_name":"2025-05-21T22:28:20Z/cfg/oplog","start_ts":1747866501,"status":"dumpDone","iscs":true,"last_transition_ts":1747866504,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":21},"node":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866504,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866501,"last_transition_ts":1747866505,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866516,"I":4},"status":"dumpDone","conditions":[{"timestamp":1747866501,"status":"starting"},{"timestamp":1747866504,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:28:41.504Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "76e38de8-25d0-489e-ba0e-445378202d59"} 2025-05-21T22:28:46.504Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782"} 2025-05-21T22:28:46.579Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:46.579Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782", "currentJob": {"Name":"backup-minio","Type":0}} 2025-05-21T22:28:46.655Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782"} 2025-05-21T22:28:46.656Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782", "backup": "backup-minio", "pbmName": "2025-05-21T22:28:20Z", "meta": {"type":"logical","opid":"682e53846f39cb2ec07effc8","name":"2025-05-21T22:28:20Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-21T22:28:20Z/rs0/metadata.json","oplog_name":"2025-05-21T22:28:20Z/rs0/oplog","start_ts":1747866501,"status":"done","last_transition_ts":1747866523,"first_write_ts":{"T":1747866494,"I":12},"last_write_ts":{"T":1747866505,"I":9},"node":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"},{"timestamp":1747866523,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-21T22:28:20Z/cfg/metadata.json","oplog_name":"2025-05-21T22:28:20Z/cfg/oplog","start_ts":1747866501,"status":"done","iscs":true,"last_transition_ts":1747866525,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":21},"node":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866504,"status":"dumpDone"},{"timestamp":1747866525,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":90319,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866501,"last_transition_ts":1747866505,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":9},"hb":{"T":1747866526,"I":4},"status":"dumpDone","conditions":[{"timestamp":1747866501,"status":"starting"},{"timestamp":1747866504,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:28:46.660Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "17f19ff7-f0ae-460a-9573-1c9697fe1782"} 2025-05-21T22:28:51.660Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0"} 2025-05-21T22:28:51.729Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:51.729Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0", "currentJob": {"Name":"backup-minio","Type":0}} 2025-05-21T22:28:51.798Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0"} 2025-05-21T22:28:51.801Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0", "backup": "backup-minio", "pbmName": "2025-05-21T22:28:20Z", "meta": {"type":"logical","opid":"682e53846f39cb2ec07effc8","name":"2025-05-21T22:28:20Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-21T22:28:20Z/rs0/metadata.json","oplog_name":"2025-05-21T22:28:20Z/rs0/oplog","start_ts":1747866501,"status":"done","last_transition_ts":1747866523,"first_write_ts":{"T":1747866494,"I":12},"last_write_ts":{"T":1747866505,"I":9},"node":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"},{"timestamp":1747866523,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-21T22:28:20Z/cfg/metadata.json","oplog_name":"2025-05-21T22:28:20Z/cfg/oplog","start_ts":1747866501,"status":"done","iscs":true,"last_transition_ts":1747866525,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":21},"node":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","conditions":[{"timestamp":1747866501,"status":"running"},{"timestamp":1747866504,"status":"dumpDone"},{"timestamp":1747866525,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":90319,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866501,"last_transition_ts":1747866526,"first_write_ts":{"T":1747866501,"I":30},"last_write_ts":{"T":1747866505,"I":9},"hb":{"T":1747866526,"I":4},"status":"done","conditions":[{"timestamp":1747866501,"status":"starting"},{"timestamp":1747866504,"status":"running"},{"timestamp":1747866505,"status":"dumpDone"},{"timestamp":1747866526,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":"some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:28:51.803Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0", "previous": "running", "current": "ready"} 2025-05-21T22:28:51.817Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T22:28:51.832Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "7db74130-7d4d-46bd-9296-6f37ac31cce0"} 2025-05-21T22:28:51.832Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "c376bf2b-4507-4216-9053-499d65ea7e2e"} 2025-05-21T22:28:51.832Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "c376bf2b-4507-4216-9053-499d65ea7e2e"} 2025-05-21T22:28:54.624Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96"} 2025-05-21T22:28:54.716Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:54.716Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:28:54.808Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96"} 2025-05-21T22:28:54.868Z DEBUG Added finalizer {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "finalizer": "internal.percona.com/release-lock"} 2025-05-21T22:28:54.868Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "backup": "backup-gcp-cs", "storage": "gcp-cs"} 2025-05-21T22:28:54.869Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "backup": "backup-gcp-cs", "storage": "gcp-cs", "backupCmd": "backup [name: 2025-05-21T22:28:54Z, compression: gzip (level: default)] ", "profile": "gcp-cs"} 2025-05-21T22:28:54.878Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96", "previous": "", "current": "requested"} 2025-05-21T22:28:54.906Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a0839329-da8e-4b8b-82c0-4bad7e722f96"} 2025-05-21T22:28:54.906Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc"} 2025-05-21T22:28:54.984Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:54.984Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:28:55.064Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc"} 2025-05-21T22:28:55.064Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc", "backup": "backup-gcp-cs", "storage": "gcp-cs"} 2025-05-21T22:28:55.064Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc", "backup": "backup-gcp-cs", "storage": "gcp-cs", "backupCmd": "backup [name: 2025-05-21T22:28:55Z, compression: gzip (level: default)] ", "profile": "gcp-cs"} 2025-05-21T22:28:55.071Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc", "previous": "", "current": "requested"} 2025-05-21T22:28:55.100Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "04c194e3-61df-48ac-89ce-789d477f6bcc"} 2025-05-21T22:28:55.100Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae"} 2025-05-21T22:28:55.222Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:55.222Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:28:55.329Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae"} 2025-05-21T22:28:55.331Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866535,"I":45},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:28:55.334Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "aa818ff0-5c7d-45da-a02c-cd0fcd4f9bae"} 2025-05-21T22:28:56.833Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "d887b5f5-9e8d-4de9-8966-08b833d6196d"} 2025-05-21T22:28:56.833Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-minio", "reconcileID": "d887b5f5-9e8d-4de9-8966-08b833d6196d"} 2025-05-21T22:28:59.906Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723"} 2025-05-21T22:28:59.977Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:28:59.977Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:00.053Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723"} 2025-05-21T22:29:00.055Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866535,"I":45},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-cfg-2.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017","some-name-rs0-2.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:00.058Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "37e00f8a-d1ef-42e0-a5b0-3dd4167b6723"} 2025-05-21T22:29:05.059Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be"} 2025-05-21T22:29:05.134Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:05.134Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:05.215Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be"} 2025-05-21T22:29:05.228Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:05.234Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "14d3d267-3f90-494a-9288-28d1deddb6be"} 2025-05-21T22:29:10.237Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89"} 2025-05-21T22:29:10.320Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:10.320Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:10.425Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89"} 2025-05-21T22:29:10.430Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:10.433Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f7aa931a-33eb-4d20-8819-672cd7de9b89"} 2025-05-21T22:29:15.434Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248"} 2025-05-21T22:29:15.514Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:15.514Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:15.602Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248"} 2025-05-21T22:29:15.603Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:15.607Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "94a8e762-ee85-4b16-bda6-621f96075248"} 2025-05-21T22:29:20.608Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd"} 2025-05-21T22:29:20.701Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:20.701Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:20.774Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd"} 2025-05-21T22:29:20.776Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:20.779Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "d3fffb94-a1fe-4b3e-b14d-96e7d58e6bdd"} 2025-05-21T22:29:25.780Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06"} 2025-05-21T22:29:25.853Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:25.853Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:25.922Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06"} 2025-05-21T22:29:25.924Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:25.926Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7b861ed4-fe5b-4ba2-8c13-9a5c0b33db06"} 2025-05-21T22:29:30.926Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072"} 2025-05-21T22:29:30.997Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:30.997Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:31.068Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072"} 2025-05-21T22:29:31.069Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:31.071Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "77c4155e-35d0-4389-b4b3-76f848ba0072"} 2025-05-21T22:29:36.071Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9"} 2025-05-21T22:29:36.138Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:36.138Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:36.212Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9"} 2025-05-21T22:29:36.213Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:36.216Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "cfe86fe7-9fa7-4741-a34b-da1795f869b9"} 2025-05-21T22:29:41.216Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71"} 2025-05-21T22:29:41.288Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:41.288Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:41.359Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71"} 2025-05-21T22:29:41.360Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:41.362Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4e7aab3e-11ab-4b06-92df-42f8a5b58b71"} 2025-05-21T22:29:46.363Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6"} 2025-05-21T22:29:46.454Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:46.454Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:46.523Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6"} 2025-05-21T22:29:46.526Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:46.528Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "4a3c6a51-6c99-45ed-ba0a-8895b8be66d6"} 2025-05-21T22:29:51.528Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29"} 2025-05-21T22:29:51.604Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:51.604Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:51.677Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29"} 2025-05-21T22:29:51.678Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:51.680Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "0c3d9561-841a-411a-9f18-d8b9a81c5d29"} 2025-05-21T22:29:56.680Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51"} 2025-05-21T22:29:56.744Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:29:56.744Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:29:56.820Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51"} 2025-05-21T22:29:56.821Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:29:56.823Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "eb6f14b6-b483-4ed4-87a4-ee0e05db5d51"} 2025-05-21T22:30:01.824Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab"} 2025-05-21T22:30:01.893Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:01.893Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:01.965Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab"} 2025-05-21T22:30:01.966Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:01.968Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "fd0cc1ba-877d-40d6-bab6-b0caf93f2dab"} 2025-05-21T22:30:06.969Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2"} 2025-05-21T22:30:07.039Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:07.039Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:07.110Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2"} 2025-05-21T22:30:07.111Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:07.114Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "a327b71d-07bd-4d53-b2f6-6dcf8191f6f2"} 2025-05-21T22:30:12.118Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100"} 2025-05-21T22:30:12.194Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:12.194Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:12.259Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100"} 2025-05-21T22:30:12.261Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:12.264Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "3eec79c9-1479-4e89-ae56-040c0e728100"} 2025-05-21T22:30:17.265Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd"} 2025-05-21T22:30:17.348Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:17.348Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:17.435Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd"} 2025-05-21T22:30:17.437Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:17.439Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "b36276ab-399c-45d5-8398-7b0a686fc8dd"} 2025-05-21T22:30:22.439Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e"} 2025-05-21T22:30:22.515Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:22.515Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:22.590Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e"} 2025-05-21T22:30:22.595Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:22.600Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "f4f594b7-51ef-499c-bd43-99c05709915e"} 2025-05-21T22:30:27.601Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266"} 2025-05-21T22:30:27.670Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:27.670Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:27.741Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266"} 2025-05-21T22:30:27.742Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:27.744Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "36f2d5a4-bfcd-435e-8c8a-d8b3213d9266"} 2025-05-21T22:30:32.748Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37"} 2025-05-21T22:30:32.820Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:32.820Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:32.892Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37"} 2025-05-21T22:30:32.894Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:32.896Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "9e196c23-8456-4d8a-a3be-7fb318e75b37"} 2025-05-21T22:30:37.896Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71"} 2025-05-21T22:30:37.968Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:37.968Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:38.045Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71"} 2025-05-21T22:30:38.047Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:38.049Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "548870ed-bfcc-444c-b643-14a490d0bc71"} 2025-05-21T22:30:43.049Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02"} 2025-05-21T22:30:43.121Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:43.121Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:43.200Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02"} 2025-05-21T22:30:43.203Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:43.206Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "c950369f-fd0a-457e-9db9-0f8262598e02"} 2025-05-21T22:30:48.207Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015"} 2025-05-21T22:30:48.281Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:48.281Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:48.359Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015"} 2025-05-21T22:30:48.361Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:48.363Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "680b4eff-3ed1-4ea7-b78c-962bf9980015"} 2025-05-21T22:30:53.363Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9"} 2025-05-21T22:30:53.450Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:53.450Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:53.531Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9"} 2025-05-21T22:30:53.532Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:53.535Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7af3d9e6-cafb-4f8a-943b-d71b68c530f9"} 2025-05-21T22:30:58.535Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3"} 2025-05-21T22:30:58.611Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3", "cluster": "some-name", "namespace": "data-at-rest-encryption-32271"} 2025-05-21T22:30:58.611Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-21T22:30:58.679Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3"} 2025-05-21T22:30:58.681Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3", "backup": "backup-gcp-cs", "pbmName": "2025-05-21T22:28:55Z", "meta": {"type":"logical","opid":"682e53a76f39cb2ec07f0014","name":"2025-05-21T22:28:55Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747866535,"last_transition_ts":1747866535,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747866540,"I":7},"status":"starting","conditions":[{"timestamp":1747866535,"status":"starting"}],"n":[{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""},{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-32271.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-21T22:30:58.682Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3", "previous": "requested", "current": "error"} 2025-05-21T22:30:58.696Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3", "lease": "psmdb-some-name-backup-lock"} 2025-05-21T22:30:58.706Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "7129ab18-3e80-43a5-bef6-ec5aed775ec3"} 2025-05-21T22:30:58.706Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "757e0a30-66ef-4405-8084-75d364ae4b20"} 2025-05-21T22:30:58.706Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-32271"}, "namespace": "data-at-rest-encryption-32271", "name": "backup-gcp-cs", "reconcileID": "757e0a30-66ef-4405-8084-75d364ae4b20"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-gcp-cs some-name gcp-cs s3://operator-testing/psmdb/2025-05-21T22:28:55Z logical error 2m8s backup-minio some-name minio s3://operator-testing/2025-05-21T22:28:20Z logical ready 2m16s 2m42s Backup object psmdb-backup/backup-gcp-cs is in error state. something went wrong with operator or kubernetes cluster