Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/logs/data-at-rest-encryption.log WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 + create_infra data-at-rest-encryption-15601 + local ns=data-at-rest-encryption-15601 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.nJJxuAU0T4 ++ mktemp + local LAST_ERR=/tmp/tmp.ueNZAOmwHY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nJJxuAU0T4 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.ueNZAOmwHY + rm /tmp/tmp.nJJxuAU0T4 /tmp/tmp.ueNZAOmwHY + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.kOxGuemXjy ++ mktemp + local LAST_ERR=/tmp/tmp.iPXRpwY3ge + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kOxGuemXjy customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.iPXRpwY3ge + rm /tmp/tmp.kOxGuemXjy /tmp/tmp.iPXRpwY3ge + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.LjP9HBEYra ++ mktemp + local LAST_ERR=/tmp/tmp.weMugmN14p + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LjP9HBEYra + cat /tmp/tmp.weMugmN14p + rm /tmp/tmp.LjP9HBEYra /tmp/tmp.weMugmN14p + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.elL0q7keeR ++ mktemp + local LAST_ERR=/tmp/tmp.NO4xIlJrT2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.elL0q7keeR + cat /tmp/tmp.NO4xIlJrT2 + rm /tmp/tmp.elL0q7keeR /tmp/tmp.NO4xIlJrT2 + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.GR8oCMl22G ++ mktemp + local LAST_ERR=/tmp/tmp.jKr8xzuX6d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GR8oCMl22G clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.jKr8xzuX6d + rm /tmp/tmp.GR8oCMl22G /tmp/tmp.jKr8xzuX6d + return 0 + check_crd_for_deletion PR-1897-ea494d1c + local git_tag=PR-1897-ea494d1c ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1897-ea494d1c/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sNLEa3io3r +++ mktemp ++ local LAST_ERR=/tmp/tmp.gFHDjNDddB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sNLEa3io3r ++ cat /tmp/tmp.gFHDjNDddB Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sNLEa3io3r ++ cat /tmp/tmp.gFHDjNDddB Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.sNLEa3io3r ++ cat /tmp/tmp.gFHDjNDddB Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.sNLEa3io3r ++ cat /tmp/tmp.gFHDjNDddB Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.sNLEa3io3r /tmp/tmp.gFHDjNDddB ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns ++ mktemp + kubectl_bin get ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + local LAST_OUT=/tmp/tmp.CLzvorKFB6 ++ mktemp + local LAST_OUT=/tmp/tmp.hgmdEJTsmj ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.mtK5LUA8mU + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.Y9IfN8l1ex + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hgmdEJTsmj + cat /tmp/tmp.mtK5LUA8mU + rm /tmp/tmp.hgmdEJTsmj /tmp/tmp.mtK5LUA8mU + return 0 namespace "data-at-rest-encryption-2007" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CLzvorKFB6 namespace "psmdb-operator" deleted + cat /tmp/tmp.Y9IfN8l1ex + rm /tmp/tmp.CLzvorKFB6 /tmp/tmp.Y9IfN8l1ex + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.quNoqYLPwg ++ mktemp + local LAST_ERR=/tmp/tmp.Hhv9Gd0VVI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.quNoqYLPwg + cat /tmp/tmp.Hhv9Gd0VVI + rm /tmp/tmp.quNoqYLPwg /tmp/tmp.Hhv9Gd0VVI + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.OdV02RTlt5 ++ mktemp + local LAST_ERR=/tmp/tmp.GryaW28uJB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OdV02RTlt5 namespace/psmdb-operator created + cat /tmp/tmp.GryaW28uJB + rm /tmp/tmp.OdV02RTlt5 /tmp/tmp.GryaW28uJB + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.ng7R7XHV9v +++ mktemp ++ local LAST_ERR=/tmp/tmp.kNN7s4QvKm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ng7R7XHV9v ++ cat /tmp/tmp.kNN7s4QvKm ++ rm /tmp/tmp.ng7R7XHV9v /tmp/tmp.kNN7s4QvKm ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.2TESICM20Z ++ mktemp + local LAST_ERR=/tmp/tmp.pED3mzZsGx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2TESICM20Z Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8" modified. + cat /tmp/tmp.pED3mzZsGx + rm /tmp/tmp.2TESICM20Z /tmp/tmp.pED3mzZsGx + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.F64YM0fN4g ++ mktemp + local LAST_ERR=/tmp/tmp.BLZQXyykR2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F64YM0fN4g customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.BLZQXyykR2 + rm /tmp/tmp.F64YM0fN4g /tmp/tmp.BLZQXyykR2 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: psmdb-operator^' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.XheRcu6np4 ++ mktemp + local LAST_ERR=/tmp/tmp.XTCkZZNROF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XheRcu6np4 clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.XTCkZZNROF + rm /tmp/tmp.XheRcu6np4 /tmp/tmp.XTCkZZNROF + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1897-ea494d1c") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VI4lOdzHzi ++ mktemp + local LAST_ERR=/tmp/tmp.HUvu3UubEl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VI4lOdzHzi deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.HUvu3UubEl + rm /tmp/tmp.VI4lOdzHzi /tmp/tmp.HUvu3UubEl + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fnj7v8iSFs +++ mktemp ++ local LAST_ERR=/tmp/tmp.W33qFUAVro ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fnj7v8iSFs ++ cat /tmp/tmp.W33qFUAVro ++ rm /tmp/tmp.Fnj7v8iSFs /tmp/tmp.W33qFUAVro ++ return 0 + wait_pod percona-server-mongodb-operator-564698798-22xj2 + local pod=percona-server-mongodb-operator-564698798-22xj2 + set +o xtrace waiting for pod/percona-server-mongodb-operator-564698798-22xj2 to be ready.OK + create_namespace data-at-rest-encryption-15601 + local namespace=data-at-rest-encryption-15601 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrole ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces data-at-rest-encryption-15601' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces data-at-rest-encryption-15601 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace data-at-rest-encryption-15601 --ignore-not-found + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.XsNGDgPQUE ++ mktemp + awk '{print$1}' + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.f6Ld6TCqBL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_OUT=/tmp/tmp.kA0JWlRakK ++ mktemp + local LAST_ERR=/tmp/tmp.K2vD9pLrKi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace data-at-rest-encryption-15601 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XsNGDgPQUE + cat /tmp/tmp.f6Ld6TCqBL + rm /tmp/tmp.XsNGDgPQUE /tmp/tmp.f6Ld6TCqBL + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kA0JWlRakK + cat /tmp/tmp.K2vD9pLrKi + rm /tmp/tmp.kA0JWlRakK /tmp/tmp.K2vD9pLrKi + return 0 + kubectl_bin wait --for=delete namespace data-at-rest-encryption-15601 ++ mktemp + local LAST_OUT=/tmp/tmp.GaD5w8xPYK ++ mktemp + local LAST_ERR=/tmp/tmp.q8ub1wCZux + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace data-at-rest-encryption-15601 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GaD5w8xPYK + cat /tmp/tmp.q8ub1wCZux + rm /tmp/tmp.GaD5w8xPYK /tmp/tmp.q8ub1wCZux + return 0 + desc 'create namespace data-at-rest-encryption-15601' + set +o xtrace ----------------------------------------------------------------------------------- create namespace data-at-rest-encryption-15601 ----------------------------------------------------------------------------------- + kubectl_bin create namespace data-at-rest-encryption-15601 ++ mktemp + local LAST_OUT=/tmp/tmp.a7BuDOxBFN ++ mktemp + local LAST_ERR=/tmp/tmp.U4O1rPOSw1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace data-at-rest-encryption-15601 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a7BuDOxBFN namespace/data-at-rest-encryption-15601 created + cat /tmp/tmp.U4O1rPOSw1 + rm /tmp/tmp.a7BuDOxBFN /tmp/tmp.U4O1rPOSw1 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.gvA3cniKG7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qSG7EB3Yzh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gvA3cniKG7 ++ cat /tmp/tmp.qSG7EB3Yzh ++ rm /tmp/tmp.gvA3cniKG7 /tmp/tmp.qSG7EB3Yzh ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8 --namespace=data-at-rest-encryption-15601 ++ mktemp + local LAST_OUT=/tmp/tmp.Aczek7yReO ++ mktemp + local LAST_ERR=/tmp/tmp.QZDogwdIFu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8 --namespace=data-at-rest-encryption-15601 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Aczek7yReO Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1897-ea494d1c-6-cluster8" modified. + cat /tmp/tmp.QZDogwdIFu + rm /tmp/tmp.Aczek7yReO /tmp/tmp.QZDogwdIFu + return 0 + deploy_vault + name=vault-service + desc 'install Vault vault-service' + set +o xtrace ----------------------------------------------------------------------------------- install Vault vault-service ----------------------------------------------------------------------------------- + helm uninstall vault-service Error: uninstall: Release not loaded: vault-service: release: not found + : + helm repo remove hashicorp "hashicorp" has been removed from your repositories + helm repo add hashicorp https://helm.releases.hashicorp.com "hashicorp" has been added to your repositories + destroy_vault ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter vault-service ++ tail -n1 ++ awk '-F ' '{print $2}' + local vault_ns= + desc 'destroy vault' + set +o xtrace ----------------------------------------------------------------------------------- destroy vault ----------------------------------------------------------------------------------- ++ awk '{print $1}' ++ kubectl api-resources ++ grep vault + '[' -n '' ']' ++ kubectl get crd ++ awk '{print $1}' ++ grep vault + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding vault-service-agent-injector-binding vault-service-server-binding clusterrolebinding.rbac.authorization.k8s.io "vault-service-agent-injector-binding" deleted clusterrolebinding.rbac.authorization.k8s.io "vault-service-server-binding" deleted ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep vault + timeout 30 kubectl delete clusterrole vault-service-agent-injector-clusterrole clusterrole.rbac.authorization.k8s.io "vault-service-agent-injector-clusterrole" deleted ++ kubectl get mutatingwebhookconfiguration ++ grep vault ++ awk '{print $1}' + timeout 30 kubectl delete mutatingwebhookconfiguration vault-service-agent-injector-cfg mutatingwebhookconfiguration.admissionregistration.k8s.io "vault-service-agent-injector-cfg" deleted + [[ -n '' ]] + retry 10 60 helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false + local max=10 + local delay=60 + shift 2 + local n=1 + helm install vault-service hashicorp/vault --disable-openapi-validation --set dataStorage.enabled=false NAME: vault-service LAST DEPLOYED: Mon May 19 14:19:27 2025 NAMESPACE: data-at-rest-encryption-15601 STATUS: deployed REVISION: 1 NOTES: Thank you for installing HashiCorp Vault! Now that you have deployed Vault, you should look over the docs on using Vault with Kubernetes available here: https://developer.hashicorp.com/vault/docs Your release is named vault-service. To learn more about the release, try: $ helm status vault-service $ helm get manifest vault-service + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' + grep Running + sleep 1 + grep Running + kubectl_bin get pod/vault-service-0 -o 'jsonpath={.status.phase}' Running + kubectl_bin exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json ++ mktemp + local LAST_OUT=/tmp/tmp.WNe7FMaMlq ++ mktemp + local LAST_ERR=/tmp/tmp.omoxxCO63B + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WNe7FMaMlq + cat /tmp/tmp.omoxxCO63B + rm /tmp/tmp.WNe7FMaMlq /tmp/tmp.omoxxCO63B + return 0 ++ jq -r '.unseal_keys_b64[]' + local unsealKey=eT7KJau5wy/SVCQn37cFX/+/MyyN1TY5aLfnSXfWkYI= ++ jq -r .root_token + local token=hvs.KRbLo3Z4Un9hXhCpCQEOD0Ri + kubectl_bin exec pod/vault-service-0 -- vault operator unseal eT7KJau5wy/SVCQn37cFX/+/MyyN1TY5aLfnSXfWkYI= ++ mktemp + local LAST_OUT=/tmp/tmp.GV0MB4SiDl ++ mktemp + local LAST_ERR=/tmp/tmp.7nnogrS9wf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec pod/vault-service-0 -- vault operator unseal eT7KJau5wy/SVCQn37cFX/+/MyyN1TY5aLfnSXfWkYI= + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GV0MB4SiDl Key Value --- ----- Seal Type shamir Initialized true Sealed false Total Shares 1 Threshold 1 Version 1.19.0 Build Date 2025-03-04T12:36:40Z Storage Type file Cluster Name vault-cluster-eb8dccb2 Cluster ID afd8df62-1bef-b334-3fcd-9827811646fe HA Enabled false + cat /tmp/tmp.7nnogrS9wf + rm /tmp/tmp.GV0MB4SiDl /tmp/tmp.7nnogrS9wf + return 0 + kubectl_bin exec -it pod/vault-service-0 -- sh ++ mktemp + local LAST_OUT=/tmp/tmp.XEFFlsuCJZ ++ mktemp + local LAST_ERR=/tmp/tmp.EcS0GDMYUf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec -it pod/vault-service-0 -- sh + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XEFFlsuCJZ Success! You are now authenticated. The token information displayed below is already stored in the token helper. You do NOT need to run "vault login" again. Future Vault requests will automatically use this token. Key Value --- ----- token hvs.KRbLo3Z4Un9hXhCpCQEOD0Ri token_accessor XYdj3cr8K7KIXHGFGonsZFzp token_duration ∞ token_renewable false token_policies ["root"] identity_policies [] policies ["root"] Success! Enabled the kv-v2 secrets engine at: secret/ + cat /tmp/tmp.EcS0GDMYUf Unable to use a TTY - input is not a terminal or the right kind of file + rm /tmp/tmp.XEFFlsuCJZ /tmp/tmp.EcS0GDMYUf + return 0 + kubectl_bin create secret generic vault-secret --from-literal=token=hvs.KRbLo3Z4Un9hXhCpCQEOD0Ri ++ mktemp + local LAST_OUT=/tmp/tmp.J7TUVlawzb ++ mktemp + local LAST_ERR=/tmp/tmp.X4T0C9Z1Pf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create secret generic vault-secret --from-literal=token=hvs.KRbLo3Z4Un9hXhCpCQEOD0Ri + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J7TUVlawzb secret/vault-secret created + cat /tmp/tmp.X4T0C9Z1Pf + rm /tmp/tmp.J7TUVlawzb /tmp/tmp.X4T0C9Z1Pf + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Mon May 19 14:19:59 2025 NAMESPACE: data-at-rest-encryption-15601 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.data-at-rest-encryption-15601.svc.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace data-at-rest-encryption-15601 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace data-at-rest-encryption-15601 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace data-at-rest-encryption-15601 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace data-at-rest-encryption-15601 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Km0MNNZ1Nk +++ mktemp ++ local LAST_ERR=/tmp/tmp.bnuOqaEjBb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Km0MNNZ1Nk ++ cat /tmp/tmp.bnuOqaEjBb ++ rm /tmp/tmp.Km0MNNZ1Nk /tmp/tmp.bnuOqaEjBb ++ return 0 + MINIO_POD=minio-service-8967c7f7f-6jhfl + wait_pod minio-service-8967c7f7f-6jhfl + local pod=minio-service-8967c7f7f-6jhfl + set +o xtrace waiting for pod/minio-service-8967c7f7f-6jhfl to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-15601.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.fRdjgmzlgU ++ mktemp + local LAST_ERR=/tmp/tmp.DugZMi6Rd3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.data-at-rest-encryption-15601.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fRdjgmzlgU service/minio-service created + cat /tmp/tmp.DugZMi6Rd3 + rm /tmp/tmp.fRdjgmzlgU /tmp/tmp.DugZMi6Rd3 + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.brITcnuQ9k ++ mktemp + local LAST_ERR=/tmp/tmp.kAUQO4Npmh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.brITcnuQ9k pod "aws-cli" deleted + cat /tmp/tmp.kAUQO4Npmh If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.brITcnuQ9k /tmp/tmp.kAUQO4Npmh + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ZcF7QzyD1v ++ mktemp + local LAST_ERR=/tmp/tmp.r6A6R2tt75 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZcF7QzyD1v secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.r6A6R2tt75 + rm /tmp/tmp.ZcF7QzyD1v /tmp/tmp.r6A6R2tt75 + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.vzzi10VTAy ++ mktemp + local LAST_ERR=/tmp/tmp.4nqplrM7eU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vzzi10VTAy secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.4nqplrM7eU + rm /tmp/tmp.vzzi10VTAy /tmp/tmp.4nqplrM7eU + return 0 + cluster=some-name + desc 'create PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/some-name.yml + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_OUT=/tmp/tmp.AxcpZD8Zlm + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1897-ea494d1c"' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/some-name.yml ++ mktemp + local LAST_ERR=/tmp/tmp.cr6p6qtjwK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AxcpZD8Zlm perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.cr6p6qtjwK + rm /tmp/tmp.AxcpZD8Zlm /tmp/tmp.cr6p6qtjwK + return 0 + desc 'check if all Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 false + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KBtIT3qdqI +++ mktemp ++ local LAST_ERR=/tmp/tmp.QNI8pGfROw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KBtIT3qdqI ++ cat /tmp/tmp.QNI8pGfROw ++ rm /tmp/tmp.KBtIT3qdqI /tmp/tmp.QNI8pGfROw ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jLxxy6Aksc +++ mktemp ++ local LAST_ERR=/tmp/tmp.BQCN97TTYW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jLxxy6Aksc ++ cat /tmp/tmp.BQCN97TTYW ++ rm /tmp/tmp.jLxxy6Aksc /tmp/tmp.BQCN97TTYW ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.owPiuME2ON +++ mktemp ++ local LAST_ERR=/tmp/tmp.zLgqqeRySH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.owPiuME2ON ++ cat /tmp/tmp.zLgqqeRySH ++ rm /tmp/tmp.owPiuME2ON /tmp/tmp.zLgqqeRySH ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TxfL6ngnuS +++ mktemp ++ local LAST_ERR=/tmp/tmp.ekRgdVcyuq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TxfL6ngnuS ++ cat /tmp/tmp.ekRgdVcyuq ++ rm /tmp/tmp.TxfL6ngnuS /tmp/tmp.ekRgdVcyuq ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready..OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.....OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lB0xluenbk +++ mktemp ++ local LAST_ERR=/tmp/tmp.RAFXFAsxCZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lB0xluenbk ++ cat /tmp/tmp.RAFXFAsxCZ ++ rm /tmp/tmp.lB0xluenbk /tmp/tmp.RAFXFAsxCZ ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready....OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cvbZv3A3Mp +++ mktemp ++ local LAST_ERR=/tmp/tmp.d3huJorSPn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cvbZv3A3Mp ++ cat /tmp/tmp.d3huJorSPn ++ rm /tmp/tmp.cvbZv3A3Mp /tmp/tmp.d3huJorSPn ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.iOHXyWuqNh/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0-oc.yml ']' + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-15601", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + kubectl_bin get -o yaml statefulset/some-name-rs0 ++ mktemp + local LAST_OUT=/tmp/tmp.eWbpgAMUvH ++ mktemp + local LAST_ERR=/tmp/tmp.035KbYzFgS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eWbpgAMUvH + cat /tmp/tmp.035KbYzFgS + rm /tmp/tmp.eWbpgAMUvH /tmp/tmp.035KbYzFgS + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-rs0.yml + version_gt 1.22 ++ bc -l ++ echo '1.30 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-rs0.yml /tmp/tmp.iOHXyWuqNh/statefulset_some-name-rs0.yml + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.iOHXyWuqNh/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg ++ mktemp + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("data-at-rest-encryption-15601", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.jDO8yKX4Av ++ mktemp + local LAST_ERR=/tmp/tmp.FkGU8HhcLs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jDO8yKX4Av + cat /tmp/tmp.FkGU8HhcLs + rm /tmp/tmp.jDO8yKX4Av /tmp/tmp.FkGU8HhcLs + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-cfg.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.iOHXyWuqNh/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/statefulset_some-name-cfg.yml /tmp/tmp.iOHXyWuqNh/statefulset_some-name-cfg.yml + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-15601 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-15601 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BaPbCl9NJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.8wmxLjMZvW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BaPbCl9NJa ++ cat /tmp/tmp.8wmxLjMZvW ++ rm /tmp/tmp.BaPbCl9NJa /tmp/tmp.8wmxLjMZvW ++ return 0 + local client_container=psmdb-client-66f577db5f-hmxzg + kubectl_bin exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.W1NN4oGMeZ ++ mktemp + local LAST_ERR=/tmp/tmp.BwWMphbj9o + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W1NN4oGMeZ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1cac4e19-f1d7-4d9b-a644-6048711e5c8b") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.BwWMphbj9o + rm /tmp/tmp.W1NN4oGMeZ /tmp/tmp.BwWMphbj9o + return 0 + sleep 2 + desc 'write data, read it' + set +o xtrace ----------------------------------------------------------------------------------- write data, read it ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZnLUXe6usv +++ mktemp ++ local LAST_ERR=/tmp/tmp.ssl0vJ0xEI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZnLUXe6usv ++ cat /tmp/tmp.ssl0vJ0xEI ++ rm /tmp/tmp.ZnLUXe6usv /tmp/tmp.ssl0vJ0xEI ++ return 0 + local client_container=psmdb-client-66f577db5f-hmxzg + kubectl_bin exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ci3xHri0UP ++ mktemp + local LAST_ERR=/tmp/tmp.7z4Zui8Zke + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ci3xHri0UP Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("7e43eebe-29a6-442c-8096-1c524e2ddfb7") } Percona Server for MongoDB server version: v8.0.8-3 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.7z4Zui8Zke + rm /tmp/tmp.ci3xHri0UP /tmp/tmp.7z4Zui8Zke + return 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 + local command=find + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.data-at-rest-encryption-15601 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IUmomuvzjE +++ mktemp ++ local LAST_ERR=/tmp/tmp.xT6Pw3VTOx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IUmomuvzjE ++ cat /tmp/tmp.xT6Pw3VTOx ++ rm /tmp/tmp.IUmomuvzjE /tmp/tmp.xT6Pw3VTOx ++ return 0 + local client_container=psmdb-client-66f577db5f-hmxzg + kubectl_bin exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.HbVmblIlMw ++ mktemp + local LAST_ERR=/tmp/tmp.zLrzfu5mo6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-hmxzg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.data-at-rest-encryption-15601.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HbVmblIlMw + cat /tmp/tmp.zLrzfu5mo6 + rm /tmp/tmp.HbVmblIlMw /tmp/tmp.zLrzfu5mo6 + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/compare/find.json /tmp/tmp.iOHXyWuqNh/find + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + desc 'run minio backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-minio ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + desc 'run backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- + kubectl_bin apply -f - ++ mktemp + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/backup-minio.yml + local LAST_OUT=/tmp/tmp.6vCiWYwgP6 ++ mktemp + local LAST_ERR=/tmp/tmp.M8qA0Yrttv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6vCiWYwgP6 perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.M8qA0Yrttv + rm /tmp/tmp.6vCiWYwgP6 /tmp/tmp.M8qA0Yrttv + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state.............. + '[' -z '' ']' + desc 'run minio backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run minio backup backup-gcp-cs ----------------------------------------------------------------------------------- + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + desc 'run backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1897/e2e-tests/data-at-rest-encryption/conf/backup-gcp-cs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.iqoNeVvIy7 ++ mktemp + local LAST_ERR=/tmp/tmp.ldPuae8nLW + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iqoNeVvIy7 perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.ldPuae8nLW + rm /tmp/tmp.iqoNeVvIy7 /tmp/tmp.ldPuae8nLW + return 0 + wait_backup backup-gcp-cs + local backup_name=backup-gcp-cs + local target_state=ready + set +o xtrace waiting for backup-gcp-cs to reach ready state..............................................2025-05-19T14:25:09.278Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662"} 2025-05-19T14:25:09.352Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:09.352Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:09.424Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662"} 2025-05-19T14:25:09.426Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:09.427Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "3a43d0e0-49e0-40dc-9946-b353b1333662"} 2025-05-19T14:25:14.428Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301"} 2025-05-19T14:25:14.499Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:14.499Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:14.573Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301"} 2025-05-19T14:25:14.575Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:14.577Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "7b9ba9b2-ac74-47e2-8b5c-42970acd8301"} 2025-05-19T14:25:19.577Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7"} 2025-05-19T14:25:19.688Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:19.688Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:19.763Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7"} 2025-05-19T14:25:19.764Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:19.766Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0cf51341-3468-4f75-9113-97d82411f5d7"} 2025-05-19T14:25:24.767Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc"} 2025-05-19T14:25:24.839Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:24.839Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:24.908Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc"} 2025-05-19T14:25:24.909Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:24.912Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "514d66ea-d323-490b-8817-766208f12bcc"} 2025-05-19T14:25:29.912Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9"} 2025-05-19T14:25:29.986Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:29.986Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:30.076Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9"} 2025-05-19T14:25:30.078Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:30.079Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "b7ca4054-9fdf-4306-b243-af733eda28f9"} 2025-05-19T14:25:35.080Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799"} 2025-05-19T14:25:35.148Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:35.148Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:35.221Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799"} 2025-05-19T14:25:35.223Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:35.225Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "59268624-c134-4e6c-bba4-ae42dc4c1799"} 2025-05-19T14:25:40.226Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f"} 2025-05-19T14:25:40.304Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:40.304Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:40.378Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f"} 2025-05-19T14:25:40.379Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:40.381Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5c94194e-b0ab-4c7c-acc3-1de79e38777f"} 2025-05-19T14:25:45.381Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb"} 2025-05-19T14:25:45.450Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:45.450Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:45.517Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb"} 2025-05-19T14:25:45.519Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:45.521Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "96b4c0ce-035a-43b0-9811-baaba9d429eb"} 2025-05-19T14:25:50.523Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138"} 2025-05-19T14:25:50.609Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:50.609Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:50.716Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138"} 2025-05-19T14:25:50.717Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:50.719Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "627f7933-8702-453d-a65b-8e6b37a81138"} 2025-05-19T14:25:55.720Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624"} 2025-05-19T14:25:55.793Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:25:55.793Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:25:55.864Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624"} 2025-05-19T14:25:55.865Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:25:55.867Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "846ffe49-0fc7-4a44-a9ed-df5e71c90624"} 2025-05-19T14:26:00.868Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816"} 2025-05-19T14:26:00.939Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:00.939Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:01.006Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816"} 2025-05-19T14:26:01.009Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:01.012Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "40ccead0-bce8-4632-8a3c-71e1b3abe816"} 2025-05-19T14:26:06.013Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0"} 2025-05-19T14:26:06.077Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:06.077Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:06.165Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0"} 2025-05-19T14:26:06.167Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:06.169Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "aef16eab-480a-4c34-81d0-dd865a28d6e0"} 2025-05-19T14:26:11.170Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51"} 2025-05-19T14:26:11.247Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:11.247Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:11.317Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51"} 2025-05-19T14:26:11.318Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:11.320Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5fc68b19-b923-462b-a98f-9cc11fe76f51"} 2025-05-19T14:26:16.320Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034"} 2025-05-19T14:26:16.399Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:16.399Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:16.478Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034"} 2025-05-19T14:26:16.479Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:16.481Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "fa9fc364-388d-4bfe-a6f7-f05929c04034"} 2025-05-19T14:26:21.481Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936"} 2025-05-19T14:26:21.554Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:21.554Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:21.665Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936"} 2025-05-19T14:26:21.667Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:21.671Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "a6d976d2-0423-43e8-970e-ff21c5909936"} 2025-05-19T14:26:26.671Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb"} 2025-05-19T14:26:26.755Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb", "cluster": "some-name", "namespace": "data-at-rest-encryption-15601"} 2025-05-19T14:26:26.755Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-19T14:26:26.824Z INFO Acquiring the backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb"} 2025-05-19T14:26:26.826Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb", "backup": "backup-gcp-cs", "pbmName": "2025-05-19T14:24:23Z", "meta": {"type":"logical","opid":"682b3f1712ecd8acadd0ed02","name":"2025-05-19T14:24:23Z","shardRemap":{"cfg":"config"},"replsets":[],"compression":"gzip","store":{"name":"gcp-cs","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"https://storage.googleapis.com","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"8.0.8-3","fcv":"8.0","start_ts":1747664663,"last_transition_ts":1747664663,"first_write_ts":{"T":1,"I":1},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747664668,"I":26},"status":"starting","conditions":[{"timestamp":1747664663,"status":"starting"}],"n":[{"rs":"rs0","n":["some-name-rs0-0.some-name-rs0.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""},{"rs":"cfg","n":["some-name-cfg-0.some-name-cfg.data-at-rest-encryption-15601.svc.cluster.local:27017"],"ack":""}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-19T14:26:26.827Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb", "previous": "requested", "current": "error"} 2025-05-19T14:26:26.841Z INFO Releasing backup lock {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb", "lease": "psmdb-some-name-backup-lock"} 2025-05-19T14:26:26.856Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "0d9f1ee5-0046-44eb-9e85-22c4fe5f41eb"} 2025-05-19T14:26:26.856Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5ba4fd6e-35d1-4fab-904c-888802cea611"} 2025-05-19T14:26:26.857Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"data-at-rest-encryption-15601"}, "namespace": "data-at-rest-encryption-15601", "name": "backup-gcp-cs", "reconcileID": "5ba4fd6e-35d1-4fab-904c-888802cea611"} Backup object psmdb-backup/backup-gcp-cs is in state. something went wrong with operator or kubernetes cluster