Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/logs/demand-backup-physical-gcp-native.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + '[' -n '' ']' + create_infra demand-backup-physical-gcp-native-3281 + local ns=demand-backup-physical-gcp-native-3281 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.Yg2GTc1qCE ++ mktemp + local LAST_ERR=/tmp/tmp.hXQpgWoIJ6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Yg2GTc1qCE customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.hXQpgWoIJ6 + rm /tmp/tmp.Yg2GTc1qCE /tmp/tmp.hXQpgWoIJ6 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.OTdBynjAvu ++ mktemp + local LAST_ERR=/tmp/tmp.zwrjHIIrg3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OTdBynjAvu customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.zwrjHIIrg3 + rm /tmp/tmp.OTdBynjAvu /tmp/tmp.zwrjHIIrg3 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.FaBx2WBvYS ++ mktemp + local LAST_ERR=/tmp/tmp.IEUpZExS8d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FaBx2WBvYS + cat /tmp/tmp.IEUpZExS8d + rm /tmp/tmp.FaBx2WBvYS /tmp/tmp.IEUpZExS8d + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.9a4xSU39NE ++ mktemp + local LAST_ERR=/tmp/tmp.6ZZTTpH4Nv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9a4xSU39NE + cat /tmp/tmp.6ZZTTpH4Nv + rm /tmp/tmp.9a4xSU39NE /tmp/tmp.6ZZTTpH4Nv + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.0dosvR8Syi ++ mktemp + local LAST_ERR=/tmp/tmp.ijUp9pal6x + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0dosvR8Syi clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.ijUp9pal6x + rm /tmp/tmp.0dosvR8Syi /tmp/tmp.ijUp9pal6x + return 0 + check_crd_for_deletion PR-2247-091270b4 + local git_tag=PR-2247-091270b4 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2247-091270b4/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U0SGp9qLxy +++ mktemp ++ local LAST_ERR=/tmp/tmp.uuEYXEfxgj ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.U0SGp9qLxy ++ cat /tmp/tmp.uuEYXEfxgj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.U0SGp9qLxy ++ cat /tmp/tmp.uuEYXEfxgj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.U0SGp9qLxy ++ cat /tmp/tmp.uuEYXEfxgj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.U0SGp9qLxy ++ cat /tmp/tmp.uuEYXEfxgj Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.U0SGp9qLxy /tmp/tmp.uuEYXEfxgj ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + awk '{print$1}' + kubectl_bin delete namespace psmdb-operator --ignore-not-found + local LAST_OUT=/tmp/tmp.UeH3I3MKjo ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.75OYaT4Dd4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.siWKzf0Wod + for i in $(seq 0 2) + set +e + kubectl get ns ++ mktemp + local LAST_ERR=/tmp/tmp.UK8CVk6ewK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UeH3I3MKjo + cat /tmp/tmp.75OYaT4Dd4 + rm /tmp/tmp.UeH3I3MKjo /tmp/tmp.75OYaT4Dd4 + return 0 namespace "demand-backup-physical-gcp-native-7643" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.siWKzf0Wod namespace "psmdb-operator" deleted + cat /tmp/tmp.UK8CVk6ewK + rm /tmp/tmp.siWKzf0Wod /tmp/tmp.UK8CVk6ewK + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.oSi1dwarMY ++ mktemp + local LAST_ERR=/tmp/tmp.yquvxFM8Vq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oSi1dwarMY + cat /tmp/tmp.yquvxFM8Vq + rm /tmp/tmp.oSi1dwarMY /tmp/tmp.yquvxFM8Vq + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.SpvmyAIeCh ++ mktemp + local LAST_ERR=/tmp/tmp.5vZkOQQJS8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SpvmyAIeCh namespace/psmdb-operator created + cat /tmp/tmp.5vZkOQQJS8 + rm /tmp/tmp.SpvmyAIeCh /tmp/tmp.5vZkOQQJS8 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.288UZcBFvI +++ mktemp ++ local LAST_ERR=/tmp/tmp.ELPQXjrt9l ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.288UZcBFvI ++ cat /tmp/tmp.ELPQXjrt9l ++ rm /tmp/tmp.288UZcBFvI /tmp/tmp.ELPQXjrt9l ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ClHmwiZQV7 ++ mktemp + local LAST_ERR=/tmp/tmp.rLraDe0KSs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ClHmwiZQV7 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12" modified. + cat /tmp/tmp.rLraDe0KSs + rm /tmp/tmp.ClHmwiZQV7 /tmp/tmp.rLraDe0KSs + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2247-091270b4' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2247-091270b4 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fkMOWuPHqc ++ mktemp + local LAST_ERR=/tmp/tmp.a5Q37cTNJJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fkMOWuPHqc customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.a5Q37cTNJJ + rm /tmp/tmp.fkMOWuPHqc /tmp/tmp.a5Q37cTNJJ + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.PhLpHPIyxE ++ mktemp + local LAST_ERR=/tmp/tmp.NCEe4UNZA4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PhLpHPIyxE clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.NCEe4UNZA4 + rm /tmp/tmp.PhLpHPIyxE /tmp/tmp.NCEe4UNZA4 + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2247-091270b4") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ccOmeGToGI ++ mktemp + local LAST_ERR=/tmp/tmp.7pBSpdul52 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ccOmeGToGI deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.7pBSpdul52 + rm /tmp/tmp.ccOmeGToGI /tmp/tmp.7pBSpdul52 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.ha6r6OSmAB +++ mktemp ++ local LAST_ERR=/tmp/tmp.zcNnAdEO34 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ha6r6OSmAB ++ cat /tmp/tmp.zcNnAdEO34 ++ rm /tmp/tmp.ha6r6OSmAB /tmp/tmp.zcNnAdEO34 ++ return 0 + wait_operator_pod percona-server-mongodb-operator-64bc9b5fc9-xqpsd + local pod=percona-server-mongodb-operator-64bc9b5fc9-xqpsd + set +o xtrace waiting for pod/percona-server-mongodb-operator-64bc9b5fc9-xqpsd to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.y9HhSiw308 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zp7GwdIPv7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.y9HhSiw308 ++ cat /tmp/tmp.zp7GwdIPv7 ++ rm /tmp/tmp.y9HhSiw308 /tmp/tmp.zp7GwdIPv7 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-xqpsd ++ mktemp + local LAST_OUT=/tmp/tmp.Bio4jsdV8q ++ mktemp + local LAST_ERR=/tmp/tmp.feqXbiFDdl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-xqpsd + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bio4jsdV8q + cat /tmp/tmp.feqXbiFDdl + rm /tmp/tmp.Bio4jsdV8q /tmp/tmp.feqXbiFDdl + return 0 2026-02-19T13:18:40.968Z INFO setup Manager starting up {"gitCommit": "091270b48773244359d74f74a10340fc1c0cff51", "gitBranch": "PR-2247-091270b4", "buildTime": "", "goVersion": "go1.25.7", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-gcp-native-3281 + local namespace=demand-backup-physical-gcp-native-3281 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + awk '{print$1}' + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-physical-gcp-native-3281' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-gcp-native-3281 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-gcp-native-3281 --ignore-not-found ++ mktemp + xargs kubectl delete ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.kTdOxeHOP6 ++ mktemp + local LAST_OUT=/tmp/tmp.CCt7710UFf ++ mktemp + local LAST_ERR=/tmp/tmp.6KCbu6uPZ2 + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.lxh0zV1XIC + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + for i in $(seq 0 2) + set +e + set +e + kubectl get ns + kubectl delete namespace demand-backup-physical-gcp-native-3281 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kTdOxeHOP6 + cat /tmp/tmp.lxh0zV1XIC + rm /tmp/tmp.kTdOxeHOP6 /tmp/tmp.lxh0zV1XIC + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CCt7710UFf + cat /tmp/tmp.6KCbu6uPZ2 + rm /tmp/tmp.CCt7710UFf /tmp/tmp.6KCbu6uPZ2 + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-gcp-native-3281 ++ mktemp + local LAST_OUT=/tmp/tmp.mIAb1tXckq ++ mktemp + local LAST_ERR=/tmp/tmp.Hs6EPKUUiI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-physical-gcp-native-3281 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mIAb1tXckq + cat /tmp/tmp.Hs6EPKUUiI + rm /tmp/tmp.mIAb1tXckq /tmp/tmp.Hs6EPKUUiI + return 0 + desc 'create namespace demand-backup-physical-gcp-native-3281' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-gcp-native-3281 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-gcp-native-3281 ++ mktemp + local LAST_OUT=/tmp/tmp.8B6KeBCfkZ ++ mktemp + local LAST_ERR=/tmp/tmp.Mx0MAuTmBB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-physical-gcp-native-3281 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8B6KeBCfkZ namespace/demand-backup-physical-gcp-native-3281 created + cat /tmp/tmp.Mx0MAuTmBB + rm /tmp/tmp.8B6KeBCfkZ /tmp/tmp.Mx0MAuTmBB + return 0 + set_kube_ctx demand-backup-physical-gcp-native-3281 + local namespace=demand-backup-physical-gcp-native-3281 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.eaIlUfjiWm +++ mktemp ++ local LAST_ERR=/tmp/tmp.ay2fMh6HQo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eaIlUfjiWm ++ cat /tmp/tmp.ay2fMh6HQo ++ rm /tmp/tmp.eaIlUfjiWm /tmp/tmp.ay2fMh6HQo ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12 --namespace=demand-backup-physical-gcp-native-3281 ++ mktemp + local LAST_OUT=/tmp/tmp.4buNg5kZT8 ++ mktemp + local LAST_ERR=/tmp/tmp.FWWeSIWhUs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12 --namespace=demand-backup-physical-gcp-native-3281 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4buNg5kZT8 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2247-091270b4-5-cluster12" modified. + cat /tmp/tmp.FWWeSIWhUs + rm /tmp/tmp.4buNg5kZT8 /tmp/tmp.FWWeSIWhUs + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kda3wFT6wM ++ mktemp + local LAST_ERR=/tmp/tmp.2DEvSGgl94 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kda3wFT6wM secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.2DEvSGgl94 + rm /tmp/tmp.kda3wFT6wM /tmp/tmp.2DEvSGgl94 + return 0 + desc 'Testing on not sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on not sharded cluster ----------------------------------------------------------------------------------- + echo 'Creating PSMDB cluster' Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UmF3pA2ZiL ++ mktemp + local LAST_ERR=/tmp/tmp.8TfM6zIMdC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UmF3pA2ZiL secret/some-users created + cat /tmp/tmp.8TfM6zIMdC + rm /tmp/tmp.UmF3pA2ZiL /tmp/tmp.8TfM6zIMdC + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/some-name.yml + kubectl_bin apply -f - + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2247-091270b4"' + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-physical-gcp-native-3281/g ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.MNsko9ldQU ++ mktemp + local LAST_ERR=/tmp/tmp.tcrMkX0eHR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MNsko9ldQU perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.tcrMkX0eHR + rm /tmp/tmp.MNsko9ldQU /tmp/tmp.tcrMkX0eHR + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.qmADlNT1bt ++ mktemp + local LAST_ERR=/tmp/tmp.E1bK0b76xi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qmADlNT1bt deployment.apps/psmdb-client created + cat /tmp/tmp.E1bK0b76xi + rm /tmp/tmp.qmADlNT1bt /tmp/tmp.E1bK0b76xi + return 0 + echo 'Check if all pods started' Check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VKBL1gzkJ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cBcvNqpwsh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VKBL1gzkJ2 ++ cat /tmp/tmp.cBcvNqpwsh ++ rm /tmp/tmp.VKBL1gzkJ2 /tmp/tmp.cBcvNqpwsh ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RHep2HlgAD +++ mktemp ++ local LAST_ERR=/tmp/tmp.S3jNgPXDYF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RHep2HlgAD ++ cat /tmp/tmp.S3jNgPXDYF ++ rm /tmp/tmp.RHep2HlgAD /tmp/tmp.S3jNgPXDYF ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ac0ftvZHRo +++ mktemp ++ local LAST_ERR=/tmp/tmp.nzEyFMpQr3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ac0ftvZHRo ++ cat /tmp/tmp.nzEyFMpQr3 ++ rm /tmp/tmp.Ac0ftvZHRo /tmp/tmp.nzEyFMpQr3 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............... + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fJtsqfnB2Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.SXF9BNShU4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fJtsqfnB2Y ++ cat /tmp/tmp.SXF9BNShU4 ++ rm /tmp/tmp.fJtsqfnB2Y /tmp/tmp.SXF9BNShU4 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + echo 'Writing test data' Writing test data + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-3281 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m2sy7f9u4C +++ mktemp ++ local LAST_ERR=/tmp/tmp.uIcri5lCEb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.m2sy7f9u4C ++ cat /tmp/tmp.uIcri5lCEb ++ rm /tmp/tmp.m2sy7f9u4C /tmp/tmp.uIcri5lCEb ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.YztunRV4rU ++ mktemp + local LAST_ERR=/tmp/tmp.XkvvhrRsjJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YztunRV4rU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b108e94e-c0d4-45df-892c-436acfdb6bd0") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.XkvvhrRsjJ + rm /tmp/tmp.YztunRV4rU /tmp/tmp.XkvvhrRsjJ + return 0 + sleep 1 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U0dFIlfBvQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.cNn3RbNK2a ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U0dFIlfBvQ ++ cat /tmp/tmp.cNn3RbNK2a ++ rm /tmp/tmp.U0dFIlfBvQ /tmp/tmp.cNn3RbNK2a ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.2Q6SYt8RoM ++ mktemp + local LAST_ERR=/tmp/tmp.WB2Hh1HRlJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2Q6SYt8RoM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("05480ae9-ed46-4875-a64e-ac607707b9bb") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.WB2Hh1HRlJ + rm /tmp/tmp.2Q6SYt8RoM /tmp/tmp.WB2Hh1HRlJ + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-02-19T13:22:42+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ihGYPykAXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.4AkIXuBylb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ihGYPykAXv ++ cat /tmp/tmp.4AkIXuBylb ++ rm /tmp/tmp.ihGYPykAXv /tmp/tmp.4AkIXuBylb ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.CKkLt0OHkY ++ mktemp + local LAST_ERR=/tmp/tmp.mIFE5n5K0n + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CKkLt0OHkY + cat /tmp/tmp.mIFE5n5K0n + rm /tmp/tmp.CKkLt0OHkY /tmp/tmp.mIFE5n5K0n + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.ChadtxSkvS/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-02-19T13:22:45+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281 mongodb '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3YInZ85ks6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QgPooe41Wc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3YInZ85ks6 ++ cat /tmp/tmp.QgPooe41Wc ++ rm /tmp/tmp.3YInZ85ks6 /tmp/tmp.QgPooe41Wc ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.HkeM1ayXWN ++ mktemp + local LAST_ERR=/tmp/tmp.dGP5KFOXh6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HkeM1ayXWN + cat /tmp/tmp.dGP5KFOXh6 + rm /tmp/tmp.HkeM1ayXWN /tmp/tmp.dGP5KFOXh6 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.ChadtxSkvS/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-02-19T13:22:47+0000] running db.test.find() in myApp + [[ false == \t\r\u\e ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' +++ mktemp + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ local LAST_OUT=/tmp/tmp.r7wwbsw0zJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.WnjFLgIqD2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r7wwbsw0zJ ++ cat /tmp/tmp.WnjFLgIqD2 ++ rm /tmp/tmp.r7wwbsw0zJ /tmp/tmp.WnjFLgIqD2 ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Wua5iSPASO ++ mktemp + local LAST_ERR=/tmp/tmp.asnYYk9GxA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Wua5iSPASO + cat /tmp/tmp.asnYYk9GxA + rm /tmp/tmp.Wua5iSPASO /tmp/tmp.asnYYk9GxA + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/compare/find.json /tmp/tmp.ChadtxSkvS/find + echo 'Running GCP backups' Running GCP backups + backup_name_gcp=backup-gcp-cs-sa + run_backup gcp-cs-sa backup-gcp-cs-sa physical + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=physical + log 'running backup backup-gcp-cs-sa' + set +o xtrace [2026-02-19T13:22:50+0000] running backup backup-gcp-cs-sa + yq eval '.metadata.name = "backup-gcp-cs-sa" | .spec.storageName = "gcp-cs-sa" | .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xzK13sO6wG ++ mktemp + local LAST_ERR=/tmp/tmp.czQkFwhZDw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xzK13sO6wG perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.czQkFwhZDw + rm /tmp/tmp.xzK13sO6wG /tmp/tmp.czQkFwhZDw + return 0 + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state............OK + check_backup_in_storage backup-gcp-cs-sa gcs rs0 + local backup=backup-gcp-cs-sa + local storage_type=gcs + local replset=rs0 + local file=filelist.pbm + local protocol=http + local endpoint ++ get_backup_dest backup-gcp-cs-sa ++ local backup_name=backup-gcp-cs-sa ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.Bjm6Ta2lJF +++ mktemp ++ local LAST_ERR=/tmp/tmp.3ADye4itbd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Bjm6Ta2lJF ++ cat /tmp/tmp.3ADye4itbd ++ rm /tmp/tmp.Bjm6Ta2lJF /tmp/tmp.3ADye4itbd ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z + case ${storage_type} in + endpoint=storage.googleapis.com + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z /rs0/filelist.pbm + backup_dest_gcp=operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z + obj=/rs0/filelist.pbm + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z/rs0/filelist.pbm + gsutil ls gs://operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z/rs0/filelist.pbm + echo 'Backup gs://operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z/rs0/filelist.pbm found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup-physical/2026-02-19T13:22:52Z/rs0/filelist.pbm found in gcp-cs + echo 'Drop collection' Drop collection + run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z62uYQB3Ph +++ mktemp ++ local LAST_ERR=/tmp/tmp.8cPOYLQPEY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z62uYQB3Ph ++ cat /tmp/tmp.8cPOYLQPEY ++ rm /tmp/tmp.z62uYQB3Ph /tmp/tmp.8cPOYLQPEY ++ return 0 + local client_container=psmdb-client-54c7db9946-4ng4s + kubectl_bin exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ALoYyeJQPP ++ mktemp + local LAST_ERR=/tmp/tmp.YwArDPPHq8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-54c7db9946-4ng4s -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ALoYyeJQPP Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-physical-gcp-native-3281.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8e1b7d29-4793-4f21-8933-ad37e9e02f22") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.YwArDPPHq8 + rm /tmp/tmp.ALoYyeJQPP /tmp/tmp.YwArDPPHq8 + return 0 + echo 'check backup and restore -- gcp-cs-sa' check backup and restore -- gcp-cs-sa + run_restore backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/restore.yml + log 'running restore restore-backup-gcp-cs-sa' + set +o xtrace [2026-02-19T13:23:21+0000] running restore restore-backup-gcp-cs-sa + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2247/e2e-tests/demand-backup-physical-gcp-native/conf/restore.yml + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/' + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.bVJ0A066PN ++ mktemp + local LAST_ERR=/tmp/tmp.M9RYqk8NFm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bVJ0A066PN perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created + cat /tmp/tmp.M9RYqk8NFm + rm /tmp/tmp.bVJ0A066PN /tmp/tmp.M9RYqk8NFm + return 0 + run_recovery_check backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local compare_suffix=_restore + local is_sharded= + local tls= + wait_restore backup-gcp-cs-sa some-name requested 0 3000 + local backup_name=backup-gcp-cs-sa + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "requested" .................................................. ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.BwHAUoqBOP ++ mktemp + local LAST_ERR=/tmp/tmp.ix5dTQvcM6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.BwHAUoqBOP + cat /tmp/tmp.ix5dTQvcM6 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.BwHAUoqBOP + cat /tmp/tmp.ix5dTQvcM6 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.BwHAUoqBOP + cat /tmp/tmp.ix5dTQvcM6 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 8 + cat /tmp/tmp.BwHAUoqBOP + cat /tmp/tmp.ix5dTQvcM6 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + rm /tmp/tmp.BwHAUoqBOP /tmp/tmp.ix5dTQvcM6 + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-0' failed to get /tmp/pbm-agent.log from some-name-rs0-0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.aXHtdkIOTr ++ mktemp + local LAST_ERR=/tmp/tmp.Z3pZRgW1ln + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.aXHtdkIOTr + cat /tmp/tmp.Z3pZRgW1ln Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.aXHtdkIOTr + cat /tmp/tmp.Z3pZRgW1ln Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.aXHtdkIOTr + cat /tmp/tmp.Z3pZRgW1ln Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + sleep 8 + cat /tmp/tmp.aXHtdkIOTr + cat /tmp/tmp.Z3pZRgW1ln Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) cat: /tmp/pbm-agent.log: No such file or directory command terminated with exit code 1 + rm /tmp/tmp.aXHtdkIOTr /tmp/tmp.Z3pZRgW1ln + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-1' failed to get /tmp/pbm-agent.log from some-name-rs0-1 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.W1LEsYSHZS ++ mktemp + local LAST_ERR=/tmp/tmp.UOl8O8lIRH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.W1LEsYSHZS + cat /tmp/tmp.UOl8O8lIRH Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.W1LEsYSHZS + cat /tmp/tmp.UOl8O8lIRH Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.W1LEsYSHZS + cat /tmp/tmp.UOl8O8lIRH Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 8 + cat /tmp/tmp.W1LEsYSHZS + cat /tmp/tmp.UOl8O8lIRH Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + rm /tmp/tmp.W1LEsYSHZS /tmp/tmp.UOl8O8lIRH + return 1 + echo 'failed to get /tmp/pbm-agent.log from some-name-rs0-2' failed to get /tmp/pbm-agent.log from some-name-rs0-2 + grep restore-backup-gcp-cs-sa + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.LFixPtXPKP +++ mktemp ++ local LAST_ERR=/tmp/tmp.ELX7ZIU9a6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LFixPtXPKP ++ cat /tmp/tmp.ELX7ZIU9a6 ++ rm /tmp/tmp.LFixPtXPKP /tmp/tmp.ELX7ZIU9a6 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-xqpsd ++ mktemp + local LAST_OUT=/tmp/tmp.d1NBEPb49q ++ mktemp + local LAST_ERR=/tmp/tmp.tsfqYlv3h6 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-64bc9b5fc9-xqpsd + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.d1NBEPb49q + cat /tmp/tmp.tsfqYlv3h6 + rm /tmp/tmp.d1NBEPb49q /tmp/tmp.tsfqYlv3h6 + return 0 2026-02-19T14:11:02.322Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "56c67882-9ccb-4b13-b2df-940edb3e3d74", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:02.323Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "56c67882-9ccb-4b13-b2df-940edb3e3d74", "ready": false} 2026-02-19T14:11:07.323Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "72ac424b-099c-4b6d-8423-fd5fe8c32de9", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:07.324Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "72ac424b-099c-4b6d-8423-fd5fe8c32de9", "ready": false} 2026-02-19T14:11:12.325Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0963ab7e-3fc0-4282-8031-1a8332c622be", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:12.326Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0963ab7e-3fc0-4282-8031-1a8332c622be", "ready": false} 2026-02-19T14:11:17.326Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "56a8b70f-a18a-42fb-8765-8e998763d227", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:17.327Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "56a8b70f-a18a-42fb-8765-8e998763d227", "ready": false} 2026-02-19T14:11:22.329Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "79580454-18d0-434c-8930-ca316a55f17a", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:22.330Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "79580454-18d0-434c-8930-ca316a55f17a", "ready": false} 2026-02-19T14:11:27.330Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "9e4a7b1e-8cf4-43c6-8b3e-853220f07b7a", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:27.331Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "9e4a7b1e-8cf4-43c6-8b3e-853220f07b7a", "ready": false} 2026-02-19T14:11:32.333Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "89f93281-8981-4a9d-b4bd-b30221b9dfdb", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:32.334Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "89f93281-8981-4a9d-b4bd-b30221b9dfdb", "ready": false} 2026-02-19T14:11:37.335Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "86400d57-d632-4aff-8bd7-6f395f089411", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:37.335Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "86400d57-d632-4aff-8bd7-6f395f089411", "ready": false} 2026-02-19T14:11:42.336Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "81c48694-6bc7-4e1f-baf6-92be68321aa3", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:42.337Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "81c48694-6bc7-4e1f-baf6-92be68321aa3", "ready": false} 2026-02-19T14:11:47.338Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d799fb75-4a90-49a4-822e-d71901b75d1e", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:47.339Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d799fb75-4a90-49a4-822e-d71901b75d1e", "ready": false} 2026-02-19T14:11:52.340Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "19262431-6114-4c85-9f2f-6ae84555985d", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:52.341Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "19262431-6114-4c85-9f2f-6ae84555985d", "ready": false} 2026-02-19T14:11:57.342Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "530dceac-ad69-4269-8574-1d5ec2c0fcf3", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:11:57.343Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "530dceac-ad69-4269-8574-1d5ec2c0fcf3", "ready": false} 2026-02-19T14:12:02.344Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6323e67d-e6c2-4c46-bfe6-162c032827d2", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:02.345Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6323e67d-e6c2-4c46-bfe6-162c032827d2", "ready": false} 2026-02-19T14:12:07.346Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0cbf458e-71fa-471f-bd7f-b6085afba5d8", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:07.347Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0cbf458e-71fa-471f-bd7f-b6085afba5d8", "ready": false} 2026-02-19T14:12:12.348Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "e445af2d-d017-4a1b-9933-8a468c813245", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:12.349Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "e445af2d-d017-4a1b-9933-8a468c813245", "ready": false} 2026-02-19T14:12:17.350Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "3ed41217-67f6-4dc3-9a64-0f60bfb20cf1", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:17.351Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "3ed41217-67f6-4dc3-9a64-0f60bfb20cf1", "ready": false} 2026-02-19T14:12:22.351Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "72e2c5f6-9fc2-4f38-8bd5-e7c641708832", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:22.352Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "72e2c5f6-9fc2-4f38-8bd5-e7c641708832", "ready": false} 2026-02-19T14:12:27.353Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d2f2c44f-0111-41f3-a1a5-799412a910a8", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:27.354Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d2f2c44f-0111-41f3-a1a5-799412a910a8", "ready": false} 2026-02-19T14:12:32.354Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84fca37f-5357-49c6-97f5-d0fd778f0057", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:32.355Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "84fca37f-5357-49c6-97f5-d0fd778f0057", "ready": false} 2026-02-19T14:12:37.356Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "a3cb6d21-812a-4d75-92e9-b5314814e9c6", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:37.357Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "a3cb6d21-812a-4d75-92e9-b5314814e9c6", "ready": false} 2026-02-19T14:12:42.358Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b09925bc-d6e9-4f40-886d-4f1a1c496464", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:42.359Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b09925bc-d6e9-4f40-886d-4f1a1c496464", "ready": false} 2026-02-19T14:12:47.360Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "77002465-5641-42ae-9181-7995b4897a39", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:47.361Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "77002465-5641-42ae-9181-7995b4897a39", "ready": false} 2026-02-19T14:12:52.361Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d1078b7e-97f3-4d19-a199-fe3b3c2e79e7", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:52.362Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d1078b7e-97f3-4d19-a199-fe3b3c2e79e7", "ready": false} 2026-02-19T14:12:57.363Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6c36a15a-946b-4992-ad97-dbb43b7f9a8b", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:12:57.364Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "6c36a15a-946b-4992-ad97-dbb43b7f9a8b", "ready": false} 2026-02-19T14:13:02.365Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "48bcff60-1ae8-4475-8d5a-f107d946b19d", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:02.366Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "48bcff60-1ae8-4475-8d5a-f107d946b19d", "ready": false} 2026-02-19T14:13:07.367Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "9d86c820-a9a5-45b4-8e63-0ebeb45e831e", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:07.368Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "9d86c820-a9a5-45b4-8e63-0ebeb45e831e", "ready": false} 2026-02-19T14:13:12.369Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "36b9a5c8-0a3d-4c9b-9b64-540eebd955a9", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:12.369Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "36b9a5c8-0a3d-4c9b-9b64-540eebd955a9", "ready": false} 2026-02-19T14:13:17.370Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "4035f7bc-bf89-43b4-b81d-69cfd61239e1", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:17.371Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "4035f7bc-bf89-43b4-b81d-69cfd61239e1", "ready": false} 2026-02-19T14:13:22.372Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "082de58e-849e-49ac-a8aa-e8c3a9021628", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:22.373Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "082de58e-849e-49ac-a8aa-e8c3a9021628", "ready": false} 2026-02-19T14:13:27.374Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "22b1b88d-f4d1-47b1-bbd8-968c6e2dd3cf", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:27.375Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "22b1b88d-f4d1-47b1-bbd8-968c6e2dd3cf", "ready": false} 2026-02-19T14:13:32.376Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "e1ced876-3672-4b60-9563-93c743f49745", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:32.377Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "e1ced876-3672-4b60-9563-93c743f49745", "ready": false} 2026-02-19T14:13:37.378Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "83c2b978-9bb9-4e2a-a03e-f1f53c7bfa69", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:37.378Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "83c2b978-9bb9-4e2a-a03e-f1f53c7bfa69", "ready": false} 2026-02-19T14:13:42.379Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b2c5d66b-3d1e-4129-88ad-c667fb44d4f2", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:42.380Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b2c5d66b-3d1e-4129-88ad-c667fb44d4f2", "ready": false} 2026-02-19T14:13:47.382Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "3617c75b-83e2-435b-8436-781fee9a9525", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:47.382Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "3617c75b-83e2-435b-8436-781fee9a9525", "ready": false} 2026-02-19T14:13:52.383Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f46398f1-76f3-4351-af44-ad8c4d07cc23", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:52.384Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f46398f1-76f3-4351-af44-ad8c4d07cc23", "ready": false} 2026-02-19T14:13:57.385Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "4b867c3e-cded-4ba3-b0b4-1f530abaddab", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:13:57.386Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "4b867c3e-cded-4ba3-b0b4-1f530abaddab", "ready": false} 2026-02-19T14:14:02.387Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "51881e9e-c909-42cf-93b9-ab078f24fb59", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:02.388Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "51881e9e-c909-42cf-93b9-ab078f24fb59", "ready": false} 2026-02-19T14:14:07.388Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "eda1f68b-30f9-4416-9fad-4f7d30aaff98", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:07.389Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "eda1f68b-30f9-4416-9fad-4f7d30aaff98", "ready": false} 2026-02-19T14:14:12.389Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "7af59fef-632d-443d-8c20-c44674285e2b", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:12.390Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "7af59fef-632d-443d-8c20-c44674285e2b", "ready": false} 2026-02-19T14:14:17.391Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04d3409e-a2c0-42fd-b77c-8efae7788e5c", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:17.391Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "04d3409e-a2c0-42fd-b77c-8efae7788e5c", "ready": false} 2026-02-19T14:14:22.392Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "8f85e87c-ade3-4281-b88b-901c2a89b430", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:22.393Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "8f85e87c-ade3-4281-b88b-901c2a89b430", "ready": false} 2026-02-19T14:14:27.394Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b312fadd-61b9-404d-9294-69f470f1e2df", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:27.395Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "b312fadd-61b9-404d-9294-69f470f1e2df", "ready": false} 2026-02-19T14:14:32.396Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0b81b37e-a40f-418b-8c3e-75d8f5a4e471", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:32.397Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "0b81b37e-a40f-418b-8c3e-75d8f5a4e471", "ready": false} 2026-02-19T14:14:37.399Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ca2e4682-5837-4123-9aae-5069eede7c66", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:37.399Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ca2e4682-5837-4123-9aae-5069eede7c66", "ready": false} 2026-02-19T14:14:42.401Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "facdd1da-bea4-464a-8c18-1a3d79d2f95d", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:42.402Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "facdd1da-bea4-464a-8c18-1a3d79d2f95d", "ready": false} 2026-02-19T14:14:47.403Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "5b3795de-6127-4078-ad64-075ee22cf0d7", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:47.404Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "5b3795de-6127-4078-ad64-075ee22cf0d7", "ready": false} 2026-02-19T14:14:52.405Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "806c6690-dd4d-464e-b52d-099c26e6695c", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:52.405Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "806c6690-dd4d-464e-b52d-099c26e6695c", "ready": false} 2026-02-19T14:14:57.406Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3df8e92-9830-4341-a471-885dc95cd968", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:14:57.411Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "d3df8e92-9830-4341-a471-885dc95cd968", "ready": false} 2026-02-19T14:15:02.412Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ca12fd3d-7d1f-457f-8a58-33cc1019ed06", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:15:02.413Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "ca12fd3d-7d1f-457f-8a58-33cc1019ed06", "ready": false} 2026-02-19T14:15:07.415Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f222f557-7ca2-4401-b922-6d6eca6e0228", "cluster": "some-name", "namespace": "demand-backup-physical-gcp-native-3281"} 2026-02-19T14:15:07.415Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"}, "namespace": "demand-backup-physical-gcp-native-3281", "name": "restore-backup-gcp-cs-sa", "reconcileID": "f222f557-7ca2-4401-b922-6d6eca6e0228", "ready": false} + kubectl_bin get psmdb-restore restore-backup-gcp-cs-sa -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.g2jIpNfabf ++ mktemp + local LAST_ERR=/tmp/tmp.eOXJDOmfdJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-gcp-cs-sa -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.g2jIpNfabf apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-gcp-cs-sa","namespace":"demand-backup-physical-gcp-native-3281"},"spec":{"backupName":"backup-gcp-cs-sa","clusterName":"some-name"}} creationTimestamp: "2026-02-19T13:23:24Z" generation: 1 name: restore-backup-gcp-cs-sa namespace: demand-backup-physical-gcp-native-3281 resourceVersion: "1771507405727711012" uid: 5b73177e-e515-4e8a-af32-0df06f34a943 spec: backupName: backup-gcp-cs-sa clusterName: some-name status: state: waiting + cat /tmp/tmp.eOXJDOmfdJ + rm /tmp/tmp.g2jIpNfabf /tmp/tmp.eOXJDOmfdJ + return 0 + kubectl_bin describe psmdb-restore restore-backup-gcp-cs-sa ++ mktemp + local LAST_OUT=/tmp/tmp.pWBOmt3ACs ++ mktemp + local LAST_ERR=/tmp/tmp.zW6LtSxMqM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-gcp-cs-sa + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pWBOmt3ACs Name: restore-backup-gcp-cs-sa Namespace: demand-backup-physical-gcp-native-3281 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2026-02-19T13:23:24Z Generation: 1 Resource Version: 1771507405727711012 UID: 5b73177e-e515-4e8a-af32-0df06f34a943 Spec: Backup Name: backup-gcp-cs-sa Cluster Name: some-name Status: State: waiting Events: + cat /tmp/tmp.zW6LtSxMqM + rm /tmp/tmp.pWBOmt3ACs /tmp/tmp.zW6LtSxMqM + return 0 + echo 'Restore object restore-backup-gcp-cs-sa is in waiting state after 50 minutes.' Restore object restore-backup-gcp-cs-sa is in waiting state after 50 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1