============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[demand-backup-physical-sharded-oss-native] [00:48:26.440809] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [00:48:28.867862] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + '[' -n '' ']' + create_infra demand-backup-physical-sharded-oss-native-7616 + local ns=demand-backup-physical-sharded-oss-native-7616 + echo demand-backup-physical-sharded-oss-native-7616 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.ajaYokPoT4 ++ mktemp + local LAST_ERR=/tmp/tmp.2ThYNuBRFV + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ajaYokPoT4 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.2ThYNuBRFV + rm /tmp/tmp.ajaYokPoT4 /tmp/tmp.2ThYNuBRFV + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-oss-native-1558 backup-oss-native-sharded --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-oss-native-sharded patched + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.qCoPHqxwxO ++ mktemp + local LAST_ERR=/tmp/tmp.KkYXCm9HOs + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qCoPHqxwxO + cat /tmp/tmp.KkYXCm9HOs + rm /tmp/tmp.qCoPHqxwxO /tmp/tmp.KkYXCm9HOs + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.pA0m0Kp7WG ++ mktemp + local LAST_ERR=/tmp/tmp.IwhsYfoEEs + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pA0m0Kp7WG + cat /tmp/tmp.IwhsYfoEEs + rm /tmp/tmp.pA0m0Kp7WG /tmp/tmp.IwhsYfoEEs + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.4hB4grwwvx ++ mktemp + local LAST_ERR=/tmp/tmp.9ipKHiisze + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4hB4grwwvx + cat /tmp/tmp.9ipKHiisze + rm /tmp/tmp.4hB4grwwvx /tmp/tmp.9ipKHiisze + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.yzzaBOU9D9 ++ mktemp + local LAST_ERR=/tmp/tmp.nhY2f48ppZ + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yzzaBOU9D9 + cat /tmp/tmp.nhY2f48ppZ + rm /tmp/tmp.yzzaBOU9D9 /tmp/tmp.nhY2f48ppZ + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.s9navwRKJS ++ mktemp + local LAST_ERR=/tmp/tmp.AlcbHQeMgM + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s9navwRKJS clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.AlcbHQeMgM + rm /tmp/tmp.s9navwRKJS /tmp/tmp.AlcbHQeMgM + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zNJXyoaUBj +++ mktemp ++ local LAST_ERR=/tmp/tmp.3gYrFXIdaC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zNJXyoaUBj ++ cat /tmp/tmp.3gYrFXIdaC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zNJXyoaUBj ++ cat /tmp/tmp.3gYrFXIdaC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.zNJXyoaUBj ++ cat /tmp/tmp.3gYrFXIdaC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.zNJXyoaUBj ++ cat /tmp/tmp.3gYrFXIdaC Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.zNJXyoaUBj /tmp/tmp.3gYrFXIdaC ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + [[ -n '' ]] + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- + xargs kubectl delete ns cleaned up old namespaces psmdb-operator++ mktemp ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.vFzATv2bDg + local LAST_OUT=/tmp/tmp.xR1jj4BVpG ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.MP6h5vxH0t + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.10dpWdYyNf + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vFzATv2bDg + cat /tmp/tmp.MP6h5vxH0t + rm /tmp/tmp.vFzATv2bDg /tmp/tmp.MP6h5vxH0t + return 0 namespace "demand-backup-incremental-sharded-oss-native-1558" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xR1jj4BVpG namespace "psmdb-operator" deleted + cat /tmp/tmp.10dpWdYyNf + rm /tmp/tmp.xR1jj4BVpG /tmp/tmp.10dpWdYyNf + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.gOyDhFDHYq ++ mktemp + local LAST_ERR=/tmp/tmp.JHF1KEZm4Q + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gOyDhFDHYq + cat /tmp/tmp.JHF1KEZm4Q + rm /tmp/tmp.gOyDhFDHYq /tmp/tmp.JHF1KEZm4Q + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.molyzaDPFa ++ mktemp + local LAST_ERR=/tmp/tmp.h58MeNLMLQ + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.molyzaDPFa namespace/psmdb-operator created + cat /tmp/tmp.h58MeNLMLQ + rm /tmp/tmp.molyzaDPFa /tmp/tmp.h58MeNLMLQ + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.fAJFwrwv0S +++ mktemp ++ local LAST_ERR=/tmp/tmp.0qxc1dbSxe ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fAJFwrwv0S ++ cat /tmp/tmp.0qxc1dbSxe ++ rm /tmp/tmp.fAJFwrwv0S /tmp/tmp.0qxc1dbSxe ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ElBgpb7Hyv ++ mktemp + local LAST_ERR=/tmp/tmp.Q6cc1sqrEY + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ElBgpb7Hyv Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8" modified. + cat /tmp/tmp.Q6cc1sqrEY + rm /tmp/tmp.ElBgpb7Hyv /tmp/tmp.Q6cc1sqrEY + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.l8QZYSSXe4 ++ mktemp + local LAST_ERR=/tmp/tmp.JFpo898aB5 + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l8QZYSSXe4 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.JFpo898aB5 + rm /tmp/tmp.l8QZYSSXe4 /tmp/tmp.JFpo898aB5 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.qIMQ1edQm5 ++ mktemp + local LAST_ERR=/tmp/tmp.NEar2WnaIY + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qIMQ1edQm5 clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.NEar2WnaIY + rm /tmp/tmp.qIMQ1edQm5 /tmp/tmp.NEar2WnaIY + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WBqK5ORRph ++ mktemp + local LAST_ERR=/tmp/tmp.MOSPGlDJAU + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WBqK5ORRph deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.MOSPGlDJAU + rm /tmp/tmp.WBqK5ORRph /tmp/tmp.MOSPGlDJAU + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.8l5FWM5YDS +++ mktemp ++ local LAST_ERR=/tmp/tmp.6BLdMVoaLJ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8l5FWM5YDS ++ cat /tmp/tmp.6BLdMVoaLJ ++ rm /tmp/tmp.8l5FWM5YDS /tmp/tmp.6BLdMVoaLJ ++ return 0 + wait_operator_pod percona-server-mongodb-operator-66cd7b75cf-tnnjd + local pod=percona-server-mongodb-operator-66cd7b75cf-tnnjd + set +o xtrace waiting for pod/percona-server-mongodb-operator-66cd7b75cf-tnnjd to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.u2UCTOVFDr +++ mktemp ++ local LAST_ERR=/tmp/tmp.aNz0p3Mnl8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.u2UCTOVFDr ++ cat /tmp/tmp.aNz0p3Mnl8 ++ rm /tmp/tmp.u2UCTOVFDr /tmp/tmp.aNz0p3Mnl8 ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-tnnjd ++ mktemp + local LAST_OUT=/tmp/tmp.wdkBkw6TF3 ++ mktemp + local LAST_ERR=/tmp/tmp.M1c4nUsKD3 + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-tnnjd') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-tnnjd + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wdkBkw6TF3 + cat /tmp/tmp.M1c4nUsKD3 + rm /tmp/tmp.wdkBkw6TF3 /tmp/tmp.M1c4nUsKD3 + return 0 2026-07-17T00:49:48.985Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-physical-sharded-oss-native-7616 + local namespace=demand-backup-physical-sharded-oss-native-7616 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] ++ mktemp + desc 'cleaned up old namespaces demand-backup-physical-sharded-oss-native-7616' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-physical-sharded-oss-native-7616 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-physical-sharded-oss-native-7616 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.kZ5r0ZZwZ5 ++ mktemp + local LAST_OUT=/tmp/tmp.onnGJelHfj ++ mktemp + local LAST_ERR=/tmp/tmp.Svbc7LPXXq + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.09yZgwvBeC + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'demand-backup-physical-sharded-oss-native-7616' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-physical-sharded-oss-native-7616 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kZ5r0ZZwZ5 + cat /tmp/tmp.Svbc7LPXXq + rm /tmp/tmp.kZ5r0ZZwZ5 /tmp/tmp.Svbc7LPXXq + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.onnGJelHfj + cat /tmp/tmp.09yZgwvBeC + rm /tmp/tmp.onnGJelHfj /tmp/tmp.09yZgwvBeC + return 0 + kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-oss-native-7616 ++ mktemp + local LAST_OUT=/tmp/tmp.jyGByVimfy ++ mktemp error: resource(s) were provided, but no name was specified + local LAST_ERR=/tmp/tmp.u3Bmt61ZIr + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'demand-backup-physical-sharded-oss-native-7616') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-physical-sharded-oss-native-7616 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jyGByVimfy + cat /tmp/tmp.u3Bmt61ZIr + rm /tmp/tmp.jyGByVimfy /tmp/tmp.u3Bmt61ZIr + return 0 + desc 'create namespace demand-backup-physical-sharded-oss-native-7616' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-physical-sharded-oss-native-7616 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-physical-sharded-oss-native-7616 ++ mktemp + local LAST_OUT=/tmp/tmp.Td6zbG1Fbt ++ mktemp + local LAST_ERR=/tmp/tmp.SmjhKyIdhl + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'demand-backup-physical-sharded-oss-native-7616') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-physical-sharded-oss-native-7616 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Td6zbG1Fbt namespace/demand-backup-physical-sharded-oss-native-7616 created + cat /tmp/tmp.SmjhKyIdhl + rm /tmp/tmp.Td6zbG1Fbt /tmp/tmp.SmjhKyIdhl + return 0 + set_kube_ctx demand-backup-physical-sharded-oss-native-7616 + local namespace=demand-backup-physical-sharded-oss-native-7616 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Y1kPlVc3IK +++ mktemp ++ local LAST_ERR=/tmp/tmp.JGTp29S2VY ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Y1kPlVc3IK ++ cat /tmp/tmp.JGTp29S2VY ++ rm /tmp/tmp.Y1kPlVc3IK /tmp/tmp.JGTp29S2VY ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8 --namespace=demand-backup-physical-sharded-oss-native-7616 ++ mktemp + local LAST_OUT=/tmp/tmp.y0sMKSP9MA ++ mktemp + local LAST_ERR=/tmp/tmp.M70RrAXUy3 + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8' '--namespace=demand-backup-physical-sharded-oss-native-7616') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8 --namespace=demand-backup-physical-sharded-oss-native-7616 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.y0sMKSP9MA Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster8" modified. + cat /tmp/tmp.M70RrAXUy3 + rm /tmp/tmp.y0sMKSP9MA /tmp/tmp.M70RrAXUy3 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.CEKtdfvKlT ++ mktemp + local LAST_ERR=/tmp/tmp.GW7I2hBjgx + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CEKtdfvKlT secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/oci-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.GW7I2hBjgx + rm /tmp/tmp.CEKtdfvKlT /tmp/tmp.GW7I2hBjgx + return 0 + desc 'Testing on sharded cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing on sharded cluster ----------------------------------------------------------------------------------- + echo 'Creating PSMDB cluster' Creating PSMDB cluster + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4kzOXXG8Gv ++ mktemp + local LAST_ERR=/tmp/tmp.EqCdcQ5G1D + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4kzOXXG8Gv secret/some-users created + cat /tmp/tmp.EqCdcQ5G1D + rm /tmp/tmp.4kzOXXG8Gv /tmp/tmp.EqCdcQ5G1D + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/some-name-sharded.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/some-name-sharded.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/some-name-sharded.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.xymmxZ0a8J + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + add_arch_to_psmdb - ++ mktemp + arch_scheduling_enabled + [[ -n '' ]] + cat - + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-physical-sharded-oss-native-7616/g + local LAST_ERR=/tmp/tmp.nZKcIdRnHT + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xymmxZ0a8J perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.nZKcIdRnHT + rm /tmp/tmp.xymmxZ0a8J /tmp/tmp.nZKcIdRnHT + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ra0o0zEkC0 ++ mktemp + local LAST_ERR=/tmp/tmp.0xFW2aCFYL + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ra0o0zEkC0 deployment.apps/psmdb-client created + cat /tmp/tmp.0xFW2aCFYL + rm /tmp/tmp.ra0o0zEkC0 /tmp/tmp.0xFW2aCFYL + return 0 + echo 'Check if all pods started' Check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.akmMZFxqKv +++ mktemp ++ local LAST_ERR=/tmp/tmp.98s6YY7jcg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.akmMZFxqKv ++ cat /tmp/tmp.98s6YY7jcg ++ rm /tmp/tmp.akmMZFxqKv /tmp/tmp.98s6YY7jcg ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2AGh6fc00P +++ mktemp ++ local LAST_ERR=/tmp/tmp.BFkI1nEpE6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2AGh6fc00P ++ cat /tmp/tmp.BFkI1nEpE6 ++ rm /tmp/tmp.2AGh6fc00P /tmp/tmp.BFkI1nEpE6 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6jqixIPo1d +++ mktemp ++ local LAST_ERR=/tmp/tmp.JcWUeEshE2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6jqixIPo1d ++ cat /tmp/tmp.JcWUeEshE2 ++ rm /tmp/tmp.6jqixIPo1d /tmp/tmp.JcWUeEshE2 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready....................................................................................................................... + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yMbhEVj1vQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.chYRO8B9Se ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yMbhEVj1vQ ++ cat /tmp/tmp.chYRO8B9Se ++ rm /tmp/tmp.yMbhEVj1vQ /tmp/tmp.chYRO8B9Se ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qDxLhRvo1C +++ mktemp ++ local LAST_ERR=/tmp/tmp.dNiKfyQVla ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qDxLhRvo1C ++ cat /tmp/tmp.dNiKfyQVla ++ rm /tmp/tmp.qDxLhRvo1C /tmp/tmp.dNiKfyQVla ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QlcqZERpdm +++ mktemp ++ local LAST_ERR=/tmp/tmp.jKJPNeqh9D ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QlcqZERpdm ++ cat /tmp/tmp.jKJPNeqh9D ++ rm /tmp/tmp.QlcqZERpdm /tmp/tmp.jKJPNeqh9D ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BAzNHXtU2S +++ mktemp ++ local LAST_ERR=/tmp/tmp.at6JaGfIDu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BAzNHXtU2S ++ cat /tmp/tmp.at6JaGfIDu ++ rm /tmp/tmp.BAzNHXtU2S /tmp/tmp.at6JaGfIDu ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ypuKSV3baY +++ mktemp ++ local LAST_ERR=/tmp/tmp.FppnbuC0J9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ypuKSV3baY ++ cat /tmp/tmp.FppnbuC0J9 ++ rm /tmp/tmp.ypuKSV3baY /tmp/tmp.FppnbuC0J9 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1lJWjw7YGh +++ mktemp ++ local LAST_ERR=/tmp/tmp.lTwjZrayRi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1lJWjw7YGh ++ cat /tmp/tmp.lTwjZrayRi ++ rm /tmp/tmp.1lJWjw7YGh /tmp/tmp.lTwjZrayRi ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q8OuEyPsHw +++ mktemp ++ local LAST_ERR=/tmp/tmp.37rzp0DFLQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q8OuEyPsHw ++ cat /tmp/tmp.37rzp0DFLQ ++ rm /tmp/tmp.q8OuEyPsHw /tmp/tmp.37rzp0DFLQ ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}' ++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6BE7KsEFP3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OsVHDCAXr2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'svc' 'some-name-mongos' '-o=jsonpath={.status}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get svc some-name-mongos '-o=jsonpath={.status}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6BE7KsEFP3 ++ cat /tmp/tmp.OsVHDCAXr2 ++ rm /tmp/tmp.6BE7KsEFP3 /tmp/tmp.OsVHDCAXr2 ++ return 0 + lbEndpoint=34.133.222.68 + '[' -z 34.133.222.68 ']' + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FJnJurWhRk +++ mktemp ++ local LAST_ERR=/tmp/tmp.lbnX6bFwo7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FJnJurWhRk ++ cat /tmp/tmp.lbnX6bFwo7 ++ rm /tmp/tmp.FJnJurWhRk /tmp/tmp.lbnX6bFwo7 ++ return 0 + local client_container=psmdb-client-7fccbb9c8c-rhvqz + kubectl_bin exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.dTCkPplEaU ++ mktemp + local LAST_ERR=/tmp/tmp.opnGlCwBbc + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7fccbb9c8c-rhvqz' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dTCkPplEaU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("9f0de266-f601-4e96-a192-4800d08e93d7") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.opnGlCwBbc + rm /tmp/tmp.dTCkPplEaU /tmp/tmp.opnGlCwBbc + return 0 + sleep 1 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eKI1rOXqsc +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kn6cIWGFO7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eKI1rOXqsc ++ cat /tmp/tmp.Kn6cIWGFO7 ++ rm /tmp/tmp.eKI1rOXqsc /tmp/tmp.Kn6cIWGFO7 ++ return 0 + local client_container=psmdb-client-7fccbb9c8c-rhvqz + kubectl_bin exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9jQrkz61vJ ++ mktemp + local LAST_ERR=/tmp/tmp.FJOzEFxA8D + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7fccbb9c8c-rhvqz' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9jQrkz61vJ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("24109d7b-d88e-47a1-921d-6fef57f95d28") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.FJOzEFxA8D + rm /tmp/tmp.9jQrkz61vJ /tmp/tmp.FJOzEFxA8D + return 0 + sleep 5 + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 -sharded + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local postfix=-sharded + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T00:57:04+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5As8xi6Zkz +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZjJ6ufejR2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5As8xi6Zkz ++ cat /tmp/tmp.ZjJ6ufejR2 ++ rm /tmp/tmp.5As8xi6Zkz /tmp/tmp.ZjJ6ufejR2 ++ return 0 + local client_container=psmdb-client-7fccbb9c8c-rhvqz + kubectl_bin exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.dfdzQb6wWK ++ mktemp + local LAST_ERR=/tmp/tmp.ZyfAaF3jyJ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7fccbb9c8c-rhvqz' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dfdzQb6wWK + cat /tmp/tmp.ZyfAaF3jyJ + rm /tmp/tmp.dfdzQb6wWK /tmp/tmp.ZyfAaF3jyJ + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/compare/find-sharded.json /tmp/tmp.AzMyKKlVAt/find-sharded + echo 'Waiting 60 seconds for stable timestamp in wiredtiger' Waiting 60 seconds for stable timestamp in wiredtiger + sleep 80 + echo 'Running OSS native storage backup' Running OSS native storage backup + backup_name_oss_native=backup-oss-native-sharded + run_backup oss-native backup-oss-native-sharded physical + local storage=oss-native + local backup_name=backup-oss-native-sharded + local type=physical + log 'running backup backup-oss-native-sharded' + set +o xtrace [2026-07-17T00:58:27+0000] running backup backup-oss-native-sharded + yq eval $'.metadata.name = "backup-oss-native-sharded"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.iX9b9SFPZ5 ++ mktemp + local LAST_ERR=/tmp/tmp.9Q4qcb5Ps7 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iX9b9SFPZ5 perconaservermongodbbackup.psmdb.percona.com/backup-oss-native-sharded created + cat /tmp/tmp.9Q4qcb5Ps7 + rm /tmp/tmp.iX9b9SFPZ5 /tmp/tmp.9Q4qcb5Ps7 + return 0 + wait_backup backup-oss-native-sharded + local backup_name=backup-oss-native-sharded + local target_state=ready + set +o xtrace waiting for backup-oss-native-sharded to reach ready state......................................................OK + check_backup_in_storage backup-oss-native-sharded oss rs0 + local backup=backup-oss-native-sharded + local storage_type=oss + local replset=rs0 + local file=filelist.pbm + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native-sharded ++ local backup_name=backup-oss-native-sharded ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.SdS7qEAdZ8 ++ sed 's|oss://||' ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.hmrvduwOoX ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SdS7qEAdZ8 ++ cat /tmp/tmp.hmrvduwOoX ++ rm /tmp/tmp.SdS7qEAdZ8 /tmp/tmp.hmrvduwOoX ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native-sharded endpointUrl ++ local backup_name=backup-oss-native-sharded ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HxHcmeiUqW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eAIp57QVjN +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.HxHcmeiUqW +++ cat /tmp/tmp.eAIp57QVjN +++ rm /tmp/tmp.HxHcmeiUqW /tmp/tmp.eAIp57QVjN +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qBMgEaOna9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CYiKUQLFKk +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qBMgEaOna9 +++ cat /tmp/tmp.CYiKUQLFKk +++ rm /tmp/tmp.qBMgEaOna9 /tmp/tmp.CYiKUQLFKk +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9lZu3fkv6J +++ mktemp ++ local LAST_ERR=/tmp/tmp.LtptC48Lz7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9lZu3fkv6J ++ cat /tmp/tmp.LtptC48Lz7 ++ rm /tmp/tmp.9lZu3fkv6J /tmp/tmp.LtptC48Lz7 ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native-sharded region ++ local backup_name=backup-oss-native-sharded ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OwOCEUsEAh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0nYK36AcUB +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OwOCEUsEAh +++ cat /tmp/tmp.0nYK36AcUB +++ rm /tmp/tmp.OwOCEUsEAh /tmp/tmp.0nYK36AcUB +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8Ct1uDRBTh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6XDdfq0lvE +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8Ct1uDRBTh +++ cat /tmp/tmp.6XDdfq0lvE +++ rm /tmp/tmp.8Ct1uDRBTh /tmp/tmp.6XDdfq0lvE +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gMASnB9zgZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.lhBg50OqX2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gMASnB9zgZ ++ cat /tmp/tmp.lhBg50OqX2 ++ rm /tmp/tmp.gMASnB9zgZ /tmp/tmp.lhBg50OqX2 ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + local _orig_key_id=AKIARXP3OARBMEQ6FSEW _orig_secret=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 _orig_default_region= _orig_aws_region= + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z /rs0/filelist.pbm https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z + key=/rs0/filelist.pbm + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z/rs0/filelist.pbm + echo 'Backup operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z//rs0/filelist.pbm found in oss-native' Backup operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z//rs0/filelist.pbm found in oss-native + AWS_ACCESS_KEY_ID=AKIARXP3OARBMEQ6FSEW + AWS_SECRET_ACCESS_KEY=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 + AWS_DEFAULT_REGION= + AWS_REGION= + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + check_backup_in_storage backup-oss-native-sharded oss cfg + local backup=backup-oss-native-sharded + local storage_type=oss + local replset=cfg + local file=filelist.pbm + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native-sharded ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-oss-native-sharded ++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.iObRodIaz0 +++ mktemp ++ sed 's|oci://||' ++ sed 's|oss://||' ++ local LAST_ERR=/tmp/tmp.mH4BOLDXS0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iObRodIaz0 ++ cat /tmp/tmp.mH4BOLDXS0 ++ rm /tmp/tmp.iObRodIaz0 /tmp/tmp.mH4BOLDXS0 ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native-sharded endpointUrl ++ local backup_name=backup-oss-native-sharded ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Fxu8Ooerzg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.azTlYoFMwO +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Fxu8Ooerzg +++ cat /tmp/tmp.azTlYoFMwO +++ rm /tmp/tmp.Fxu8Ooerzg /tmp/tmp.azTlYoFMwO +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tYXWx5ARfY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ztvm8kCixH +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.tYXWx5ARfY +++ cat /tmp/tmp.ztvm8kCixH +++ rm /tmp/tmp.tYXWx5ARfY /tmp/tmp.ztvm8kCixH +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cdLxnVbZFP +++ mktemp ++ local LAST_ERR=/tmp/tmp.ndjOuO8Zn4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cdLxnVbZFP ++ cat /tmp/tmp.ndjOuO8Zn4 ++ rm /tmp/tmp.cdLxnVbZFP /tmp/tmp.ndjOuO8Zn4 ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native-sharded region ++ local backup_name=backup-oss-native-sharded ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RVotyegthX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M20Uxh2Wap +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.RVotyegthX +++ cat /tmp/tmp.M20Uxh2Wap +++ rm /tmp/tmp.RVotyegthX /tmp/tmp.M20Uxh2Wap +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native-sharded +++ local backup_name=backup-oss-native-sharded +++ kubectl_bin get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9M24nmKGiV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dQ5Z6OvcLd +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native-sharded' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native-sharded -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9M24nmKGiV +++ cat /tmp/tmp.dQ5Z6OvcLd +++ rm /tmp/tmp.9M24nmKGiV /tmp/tmp.dQ5Z6OvcLd +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ub0fGMBsfF +++ mktemp ++ local LAST_ERR=/tmp/tmp.fwA03chXPM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ub0fGMBsfF ++ cat /tmp/tmp.fwA03chXPM ++ rm /tmp/tmp.Ub0fGMBsfF /tmp/tmp.fwA03chXPM ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + local _orig_key_id=AKIARXP3OARBMEQ6FSEW _orig_secret=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 _orig_default_region= _orig_aws_region= + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z /cfg/filelist.pbm https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z + key=/cfg/filelist.pbm + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z/cfg/filelist.pbm + echo 'Backup operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z//cfg/filelist.pbm found in oss-native' Backup operator-testing/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00:58:29Z//cfg/filelist.pbm found in oss-native + AWS_ACCESS_KEY_ID=AKIARXP3OARBMEQ6FSEW + AWS_SECRET_ACCESS_KEY=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 + AWS_DEFAULT_REGION= + AWS_REGION= + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + echo 'Drop collection' Drop collection + run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WVKGNDSy8c +++ mktemp ++ local LAST_ERR=/tmp/tmp.9a4aC4DxfN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WVKGNDSy8c ++ cat /tmp/tmp.9a4aC4DxfN ++ rm /tmp/tmp.WVKGNDSy8c /tmp/tmp.9a4aC4DxfN ++ return 0 + local client_container=psmdb-client-7fccbb9c8c-rhvqz + kubectl_bin exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.4nuOICYs0Q ++ mktemp + local LAST_ERR=/tmp/tmp.4mH6w7UtSR + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-7fccbb9c8c-rhvqz' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.4nuOICYs0Q Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Error: couldn't connect to server some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017, connection attempt failed: HostNotFound: Could not find address for some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017: SocketException: Host not found (authoritative) : connect@src/mongo/shell/mongo.js:374:17 @(connect):2:6 exception: connect failed exiting with code 1 + cat /tmp/tmp.4mH6w7UtSR command terminated with exit code 1 + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-7fccbb9c8c-rhvqz -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4nuOICYs0Q Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8e386a59-bb08-46eb-bdbd-b609e7a1f862") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.4mH6w7UtSR + rm /tmp/tmp.4nuOICYs0Q /tmp/tmp.4mH6w7UtSR + return 0 + echo 'Check backup and restore -- oss-native' Check backup and restore -- oss-native + run_restore backup-oss-native-sharded + local backup_name=backup-oss-native-sharded + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/restore.yml + log 'running restore restore-backup-oss-native-sharded' + set +o xtrace [2026-07-17T01:01:49+0000] running restore restore-backup-oss-native-sharded + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-oss-native-sharded/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-oss-native-sharded/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.uyzdpT7Umd ++ mktemp + local LAST_ERR=/tmp/tmp.AAPrFFD8TG + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uyzdpT7Umd perconaservermongodbrestore.psmdb.percona.com/restore-backup-oss-native-sharded created + cat /tmp/tmp.AAPrFFD8TG + rm /tmp/tmp.uyzdpT7Umd /tmp/tmp.AAPrFFD8TG + return 0 + run_recovery_check backup-oss-native-sharded _restore_sharded sharded + local backup_name=backup-oss-native-sharded + local compare_suffix=_restore_sharded + local is_sharded=sharded + local tls= + wait_restore backup-oss-native-sharded some-name requested 0 3000 + local backup_name=backup-oss-native-sharded + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-oss-native-sharded object to be created.OK Waiting psmdb-restore/restore-backup-oss-native-sharded to reach state "requested" .......OK after 6 minutes + [[ 0 -eq 1 ]] + echo + compare_kubectl statefulset/some-name-rs0 _restore_sharded + local resource=statefulset/some-name-rs0 + local postfix=_restore_sharded + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/compare/statefulset_some-name-rs0_restore_sharded.yml + local new_result=/tmp/tmp.AzMyKKlVAt/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-oss-native-7616", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.qLtRjV6FuI ++ mktemp + local LAST_ERR=/tmp/tmp.ddRXNyDyND + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qLtRjV6FuI + cat /tmp/tmp.ddRXNyDyND + rm /tmp/tmp.qLtRjV6FuI /tmp/tmp.ddRXNyDyND + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.AzMyKKlVAt/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.AzMyKKlVAt/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.AzMyKKlVAt/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup-physical-sharded-oss-native/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.AzMyKKlVAt/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-07-17T01:08:02+0000] compare_kubectl: statefulset/some-name-rs0 OK + wait_restore backup-oss-native-sharded some-name ready 0 3000 + local backup_name=backup-oss-native-sharded + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-oss-native-sharded object to be created.OK Waiting psmdb-restore/restore-backup-oss-native-sharded to reach state "ready" .. ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-0 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-0 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.wJFIorvyyf ++ mktemp + local LAST_ERR=/tmp/tmp.D4JXlMoyXL + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-0' '--' 'cat' '/tmp/pbm-agent.log') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wJFIorvyyf 2026/07/17 01:06:02 [entrypoint] starting `pbm-agent` 2026-07-17T01:06:32.000+0000 E Exit: connect to PBM: create mongo connection: ping: server selection error: context deadline exceeded, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] } 2026/07/17 01:06:32 [entrypoint] `pbm-agent` exited with code 1 2026/07/17 01:06:32 [entrypoint] restart in 5 sec 2026/07/17 01:06:37 [entrypoint] starting `pbm-agent` 2026-07-17T01:07:07.000+0000 E Exit: connect to PBM: create mongo connection: ping: server selection error: context deadline exceeded, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] } 2026/07/17 01:07:07 [entrypoint] `pbm-agent` exited with code 1 2026/07/17 01:07:07 [entrypoint] restart in 5 sec 2026/07/17 01:07:12 [entrypoint] starting `pbm-agent` 2026-07-17T01:07:42.000+0000 E Exit: connect to PBM: create mongo connection: ping: server selection error: context deadline exceeded, current topology: { Type: Unknown, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: dial tcp 127.0.0.1:27017: connect: connection refused }, ] } 2026/07/17 01:07:42 [entrypoint] `pbm-agent` exited with code 1 2026/07/17 01:07:42 [entrypoint] restart in 5 sec 2026/07/17 01:07:47 [entrypoint] starting `pbm-agent` 2026-07-17T01:07:51.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-07-17T01:07:51.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-07-17T01:07:51.000+0000 I pbm-agent: Version: 2.15.0 Platform: linux/amd64 GitCommit: 15c9f217f91bc518b81ad901abe9331227e5c615 GitBranch: release-2.15.0 BuildTime: 2026-06-10_09:58_UTC GoVersion: go1.25.11 2026-07-17T01:07:51.000+0000 I starting PITR routine 2026-07-17T01:07:51.000+0000 I node: rs0/some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017 2026-07-17T01:07:51.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-07-17T01:07:51.000+0000 I listening for the commands 2026-07-17T01:08:00.000+0000 I got command restore [name: 2026-07-17T01:08:00.410100184Z, snapshot: 2026-07-17T00:58:29Z] , opid: 6a598070f9f2e68ecec35fbc 2026-07-17T01:08:00.000+0000 I got epoch {1784250158 1} 2026-07-17T01:08:00.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] waiting for PITR config disabled and OpLock released 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] PITR slicer stopped 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] backup: 2026-07-17T00:58:29Z 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] recovery started 2026-07-17T01:08:15.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] port: 27733 2026-07-17T01:08:15.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:16.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:17.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/cluster.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:17.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] mongod binary: mongod, version: v8.0.26-11 2026-07-17T01:08:43.000+0000 E [restore/2026-07-17T01:08:00.410100184Z] mark error during restore: get data for restore: open filelist "2026-07-17T00:58:29Z/rs0/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/rs0/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/rs0/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:08:43.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017.error" [size hint: 478 (478.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:52.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:08.000+0000 E [restore/2026-07-17T01:08:00.410100184Z] write error state `get data for restore: open filelist "2026-07-17T00:58:29Z/rs0/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/rs0/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/rs0/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout` to storage: save during split-merge mw: put object: upload failed, upload id: , cause: operation error InitiateMultipartUpload: Post "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.restore/2026-07-17T01%3A08%3A00.410100184Z/rs.rs0/node.some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local%3A27017.error?encoding-type=url&uploads": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:18.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:23.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] wait for cluster status + cat /tmp/tmp.D4JXlMoyXL Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.wJFIorvyyf /tmp/tmp.D4JXlMoyXL + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-1' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-1 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-1 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.y0M7vkVt9W ++ mktemp + local LAST_ERR=/tmp/tmp.8ydHxcE2XO + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-1' '--' 'cat' '/tmp/pbm-agent.log') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-1 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.y0M7vkVt9W 2026/07/17 01:05:27 [entrypoint] starting `pbm-agent` 2026-07-17T01:05:31.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-07-17T01:05:31.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-07-17T01:05:31.000+0000 I pbm-agent: Version: 2.15.0 Platform: linux/amd64 GitCommit: 15c9f217f91bc518b81ad901abe9331227e5c615 GitBranch: release-2.15.0 BuildTime: 2026-06-10_09:58_UTC GoVersion: go1.25.11 2026-07-17T01:05:31.000+0000 I starting PITR routine 2026-07-17T01:05:31.000+0000 I node: rs0/some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017 2026-07-17T01:05:31.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-07-17T01:05:31.000+0000 I listening for the commands 2026-07-17T01:06:32.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:06:58.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:07:24.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:07:51.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:08:00.000+0000 I got command restore [name: 2026-07-17T01:08:00.410100184Z, snapshot: 2026-07-17T00:58:29Z] , opid: 6a598070f9f2e68ecec35fbc 2026-07-17T01:08:00.000+0000 I got epoch {1784250158 1} 2026-07-17T01:08:00.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] waiting for PITR config disabled and OpLock released 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] PITR slicer stopped 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] backup: 2026-07-17T00:58:29Z 2026-07-17T01:08:15.000+0000 I [restore/2026-07-17T01:08:00.410100184Z] recovery started 2026-07-17T01:08:15.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] port: 27979 2026-07-17T01:08:15.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:16.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/rs.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:16.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/cluster.hb" [size hint: 10 (10.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:17.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] mongod binary: mongod, version: v8.0.26-11 2026-07-17T01:08:43.000+0000 E [restore/2026-07-17T01:08:00.410100184Z] mark error during restore: get data for restore: open filelist "2026-07-17T00:58:29Z/rs0/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/rs0/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/rs0/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:08:43.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] uploading ".pbm.restore/2026-07-17T01:08:00.410100184Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017.error" [size hint: 478 (478.00B); part size: 6291456 (6.00MB)] 2026-07-17T01:08:47.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:08.000+0000 E [restore/2026-07-17T01:08:00.410100184Z] write error state `get data for restore: open filelist "2026-07-17T00:58:29Z/rs0/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/rs0/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/rs0/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout` to storage: save during split-merge mw: put object: upload failed, upload id: , cause: operation error InitiateMultipartUpload: Post "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.restore/2026-07-17T01%3A08%3A00.410100184Z/rs.rs0/node.some-name-rs0-1.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local%3A27017.error?uploads&encoding-type=url": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:13.000+0000 E [agentCheckup] check storage connection: storage check failed with: file stat: list with parts for mw file stat op: fetching pbm file parts base for .pbm.init: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/.pbm.init": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout 2026-07-17T01:09:23.000+0000 D [restore/2026-07-17T01:08:00.410100184Z] wait for cluster status + cat /tmp/tmp.8ydHxcE2XO Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.y0M7vkVt9W /tmp/tmp.8ydHxcE2XO + return 0 + for pod in $(get_mongod_pods ${cluster}) + desc 'pbm-agent logs from some-name-rs0-2' + set +o xtrace ----------------------------------------------------------------------------------- pbm-agent logs from some-name-rs0-2 ----------------------------------------------------------------------------------- + kubectl_bin exec some-name-rs0-2 -- cat /tmp/pbm-agent.log ++ mktemp + local LAST_OUT=/tmp/tmp.Kfuyywn8MT ++ mktemp + local LAST_ERR=/tmp/tmp.bBX5fDsxue + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-2' '--' 'cat' '/tmp/pbm-agent.log') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-2 -- cat /tmp/pbm-agent.log + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Kfuyywn8MT 2026/07/17 01:09:37 [entrypoint] starting `pbm-agent` 2026-07-17T01:09:41.000+0000 I % _____ %%% | __ \ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` | #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| | ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_| ,((### ### %%% _____ _ (((( (### #### %%%% / ____| | | ((( ((# ###### | (___ __ _ _ _ __ _ __| | (((( (((# #### \___ \ / _` | | | |/ _` |/ _` | /(( ,((( *### ____) | (_| | |_| | (_| | (_| | //// ((( #### |_____/ \__, |\__,_|\__,_|\__,_| /// (((( #### | | /////////////(((((((((((((((((######## |_| Join @ squad.percona.com/mongodb ** Join Percona Squad! ** Participate in monthly SWAG raffles, get early access to new product features, invite-only ”ask me anything” sessions with database performance experts. Interested? Fill in the form at squad.percona.com/mongodb 2026-07-17T01:09:41.000+0000 I log options: log-path=/dev/stderr, log-level:D, log-json:false 2026-07-17T01:09:41.000+0000 I pbm-agent: Version: 2.15.0 Platform: linux/amd64 GitCommit: 15c9f217f91bc518b81ad901abe9331227e5c615 GitBranch: release-2.15.0 BuildTime: 2026-06-10_09:58_UTC GoVersion: go1.25.11 2026-07-17T01:09:41.000+0000 I starting PITR routine 2026-07-17T01:09:41.000+0000 I node: rs0/some-name-rs0-2.some-name-rs0.demand-backup-physical-sharded-oss-native-7616.svc.cluster.local:27017 2026-07-17T01:09:41.000+0000 I conn level ReadConcern: majority; WriteConcern: majority 2026-07-17T01:09:41.000+0000 I listening for the commands + cat /tmp/tmp.bBX5fDsxue Defaulted container "mongod" out of: mongod, mongo-init (init), pbm-init (init) + rm /tmp/tmp.Kfuyywn8MT /tmp/tmp.bBX5fDsxue + return 0 + grep restore-backup-oss-native-sharded + tail -100 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.INOYweA4sR +++ mktemp ++ local LAST_ERR=/tmp/tmp.lSmZYQD3nP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.INOYweA4sR ++ cat /tmp/tmp.lSmZYQD3nP ++ rm /tmp/tmp.INOYweA4sR /tmp/tmp.lSmZYQD3nP ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-tnnjd ++ mktemp + local LAST_OUT=/tmp/tmp.0s1lLzcpOh ++ mktemp + local LAST_ERR=/tmp/tmp.TWQW4p5lSf + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-tnnjd') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-tnnjd + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0s1lLzcpOh + cat /tmp/tmp.TWQW4p5lSf + rm /tmp/tmp.0s1lLzcpOh /tmp/tmp.TWQW4p5lSf + return 0 2026-07-17T01:04:06.532Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8614184f-92d2-4229-9751-43429b0aae8c", "ready": false} 2026-07-17T01:04:11.533Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8a600cd5-dd6f-4d66-92c8-d518c20a55bb", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:04:19.555Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8a600cd5-dd6f-4d66-92c8-d518c20a55bb", "ready": false} 2026-07-17T01:04:24.556Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "f57f9ac7-cce8-480f-a7f3-f3a1379356c4", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:04:32.553Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "f57f9ac7-cce8-480f-a7f3-f3a1379356c4", "ready": false} 2026-07-17T01:04:37.554Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "1055db1d-69d7-436f-b90a-d49e7da6074a", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:04:45.669Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "1055db1d-69d7-436f-b90a-d49e7da6074a", "ready": false} 2026-07-17T01:04:50.671Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a586e900-f895-4b00-92f9-cb38bb48c74f", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:04:58.792Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a586e900-f895-4b00-92f9-cb38bb48c74f", "ready": false} 2026-07-17T01:05:03.793Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "3c4ccb2b-cd7b-44e7-a299-0af355c55baf", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:05:11.841Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "3c4ccb2b-cd7b-44e7-a299-0af355c55baf", "pod": "some-name-rs0-2"} 2026-07-17T01:05:11.841Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "3c4ccb2b-cd7b-44e7-a299-0af355c55baf", "ready": false} 2026-07-17T01:05:16.841Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8a1e9443-969d-4060-9fe6-fb69379db060", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:05:24.918Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8a1e9443-969d-4060-9fe6-fb69379db060", "ready": false} 2026-07-17T01:05:29.919Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "30fd3498-41b5-4600-87b4-17f3954f5cc7", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:05:37.924Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "30fd3498-41b5-4600-87b4-17f3954f5cc7", "ready": false} 2026-07-17T01:05:42.925Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "9eaeb45a-fa5c-447a-b1f4-9799f37013b0", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:05:50.968Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "9eaeb45a-fa5c-447a-b1f4-9799f37013b0", "ready": false} 2026-07-17T01:05:55.969Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "4611b07d-f229-4a23-9397-bbac60622b21", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:06:31.009Z INFO Waiting for pbm-agent. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "4611b07d-f229-4a23-9397-bbac60622b21"} 2026-07-17T01:06:36.010Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "cf29f6c3-de2a-4cc4-858f-5bb48a8d98e1", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:07:06.014Z INFO Waiting for pbm-agent. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "cf29f6c3-de2a-4cc4-858f-5bb48a8d98e1"} 2026-07-17T01:07:11.016Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "99489726-e694-482d-bb0a-31976df0e3ee", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:07:41.020Z INFO Waiting for pbm-agent. {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "99489726-e694-482d-bb0a-31976df0e3ee"} 2026-07-17T01:07:46.021Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:07:59.018Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "pod": "some-name-rs0-0"} 2026-07-17T01:07:59.018Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "pod": "some-name-rs0-1"} 2026-07-17T01:07:59.018Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "pod": "some-name-rs0-2"} 2026-07-17T01:07:59.018Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "sts": "some-name-rs0", "replset": "rs0"} 2026-07-17T01:07:59.018Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "sts": "some-name-cfg", "replset": "cfg"} 2026-07-17T01:07:59.823Z INFO Starting restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "command": ["/opt/percona/pbm", "restore", "2026-07-17T00:58:29Z", "--out", "json", "--yes"], "pod": "some-name-rs0-0"} 2026-07-17T01:08:00.432Z INFO Restore started {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "pod": "some-name-rs0-0"} 2026-07-17T01:08:00.432Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "dcc0283b-eb9e-4d19-901f-536ad2cdb959", "previous": "waiting", "current": "requested"} 2026-07-17T01:08:00.980Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:00.982Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "pod": "some-name-rs0-1"} 2026-07-17T01:08:00.982Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "pod": "some-name-rs0-2"} 2026-07-17T01:08:00.982Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "pod": "some-name-rs0-0"} 2026-07-17T01:08:00.982Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "sts": "some-name-rs0", "replset": "rs0"} 2026-07-17T01:08:00.982Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "sts": "some-name-cfg", "replset": "cfg"} 2026-07-17T01:08:00.982Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "command": ["/opt/percona/pbm", "describe-restore", "2026-07-17T01:08:00.410100184Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-07-17T01:08:02.080Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "6e0dde8c-b00b-437e-a4dc-b2a3c0f8fae9", "status": {"type":"physical","opid":"","name":"2026-07-17T01:08:00.410100184Z","replsets":null,"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"","conditions":null,"n":null,"pbm_version":"","balancer":""}} 2026-07-17T01:08:07.081Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:07.082Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "pod": "some-name-rs0-1"} 2026-07-17T01:08:07.082Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "pod": "some-name-rs0-2"} 2026-07-17T01:08:07.082Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "pod": "some-name-rs0-0"} 2026-07-17T01:08:07.082Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "sts": "some-name-rs0", "replset": "rs0"} 2026-07-17T01:08:07.082Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "67390b08-a0e0-45da-991d-d6a3b4417240", "ready": false} 2026-07-17T01:08:12.083Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:12.085Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "pod": "some-name-rs0-0"} 2026-07-17T01:08:12.085Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "pod": "some-name-rs0-1"} 2026-07-17T01:08:12.085Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "pod": "some-name-rs0-2"} 2026-07-17T01:08:12.085Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "sts": "some-name-rs0", "replset": "rs0"} 2026-07-17T01:08:12.085Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "986fa77a-127b-494d-8219-55ac9580d592", "ready": false} 2026-07-17T01:08:17.086Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "fb3bac3f-246f-469c-a3e6-7a9dac05c729", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:17.087Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "fb3bac3f-246f-469c-a3e6-7a9dac05c729", "ready": false} 2026-07-17T01:08:22.088Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d4a06236-eb74-43c7-b67e-24cf8594d36d", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:22.090Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d4a06236-eb74-43c7-b67e-24cf8594d36d", "ready": false} 2026-07-17T01:08:27.091Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d888749c-84bc-4716-a65e-ffa08316f897", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:27.093Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d888749c-84bc-4716-a65e-ffa08316f897", "ready": false} 2026-07-17T01:08:32.094Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "462dc99c-865f-48fa-84c2-056b4e3bebeb", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:32.095Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "462dc99c-865f-48fa-84c2-056b4e3bebeb", "ready": false} 2026-07-17T01:08:37.097Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "5c62cd71-2613-4dd6-9f1d-b7a291ad5673", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:37.098Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "5c62cd71-2613-4dd6-9f1d-b7a291ad5673", "ready": false} 2026-07-17T01:08:42.098Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "4f80f3ec-f698-4a10-bf81-e46bc66e7352", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:42.099Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "4f80f3ec-f698-4a10-bf81-e46bc66e7352", "ready": false} 2026-07-17T01:08:47.101Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "52c03bda-1c0b-4c2c-8638-6c4068af18ea", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:47.103Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "52c03bda-1c0b-4c2c-8638-6c4068af18ea", "ready": false} 2026-07-17T01:08:52.103Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "ee86b08a-8e73-42e4-ae9d-9fa1b9a310e1", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:52.105Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "ee86b08a-8e73-42e4-ae9d-9fa1b9a310e1", "ready": false} 2026-07-17T01:08:57.106Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "f8f57544-b3c6-4a8c-88fc-427bb566e451", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:08:57.107Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "f8f57544-b3c6-4a8c-88fc-427bb566e451", "ready": false} 2026-07-17T01:09:02.109Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "5f96340f-98fe-4899-bd78-1b379addb754", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:02.110Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "5f96340f-98fe-4899-bd78-1b379addb754", "ready": false} 2026-07-17T01:09:07.112Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "fa4bf9b6-79a3-45e7-9646-7f7a6404385d", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:07.113Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "fa4bf9b6-79a3-45e7-9646-7f7a6404385d", "ready": false} 2026-07-17T01:09:12.114Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "68cbfcf0-3532-4155-8e64-960dd5cdea0b", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:12.115Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "68cbfcf0-3532-4155-8e64-960dd5cdea0b", "ready": false} 2026-07-17T01:09:17.117Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "832b5cc3-5792-4678-82a5-128ed7dd35ed", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:17.118Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "832b5cc3-5792-4678-82a5-128ed7dd35ed", "ready": false} 2026-07-17T01:09:22.120Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a232f507-9719-453f-8486-34d37445822a", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:22.121Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a232f507-9719-453f-8486-34d37445822a", "ready": false} 2026-07-17T01:09:27.122Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8c80595d-6c6c-4992-a610-0c83f86c6283", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:27.124Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "8c80595d-6c6c-4992-a610-0c83f86c6283", "ready": false} 2026-07-17T01:09:32.124Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "99dbbebd-215e-428b-a802-8a2035619f91", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:32.125Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "99dbbebd-215e-428b-a802-8a2035619f91", "ready": false} 2026-07-17T01:09:37.126Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "62b93e63-a6c2-4920-889f-ee16be2aa447", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:37.127Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "62b93e63-a6c2-4920-889f-ee16be2aa447", "ready": false} 2026-07-17T01:09:42.129Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d1e8b478-4972-4878-b057-871b7dc00c50", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:42.130Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "d1e8b478-4972-4878-b057-871b7dc00c50", "ready": false} 2026-07-17T01:09:47.132Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a6c1ed4f-7307-4b90-92af-0cd6bdcdf133", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:47.133Z INFO Waiting for statefulsets to be ready before restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "a6c1ed4f-7307-4b90-92af-0cd6bdcdf133", "ready": false} 2026-07-17T01:09:52.134Z DEBUG checking if restore is allowed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "cluster": "some-name", "namespace": "demand-backup-physical-sharded-oss-native-7616"} 2026-07-17T01:09:52.135Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "pod": "some-name-rs0-1"} 2026-07-17T01:09:52.135Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "pod": "some-name-rs0-2"} 2026-07-17T01:09:52.135Z DEBUG Pod is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "pod": "some-name-rs0-0"} 2026-07-17T01:09:52.135Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "sts": "some-name-rs0", "replset": "rs0"} 2026-07-17T01:09:52.136Z DEBUG Statefulset is ready for physical restore {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "sts": "some-name-cfg", "replset": "cfg"} 2026-07-17T01:09:52.136Z DEBUG Check restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "command": ["/opt/percona/pbm", "describe-restore", "2026-07-17T01:08:00.410100184Z", "--config", "/etc/pbm/pbm_config.yaml", "--out", "json"], "pod": "some-name-rs0-0"} 2026-07-17T01:09:55.413Z DEBUG PBM restore status {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "status": {"type":"physical","opid":"","name":"2026-07-17T01:08:00.410100184Z","replsets":[{"name":"rs0","start_ts":0,"status":"","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","conditions":null},{"name":"cfg","start_ts":0,"status":"error","size":0,"size_uncompressed":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"node":"","error":"get data for restore: open filelist \"2026-07-17T00:58:29Z/cfg/filelist.pbm\": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/cfg/filelist.pbm: get OSS object header: operation error HeadObject: Head \"https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/cfg/filelist.pbm\": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout","conditions":null}],"compression":"","store":{"type":""},"size":0,"size_uncompressed":0,"mongodb_version":"","fcv":"","start_ts":0,"last_transition_ts":0,"first_write_ts":{"T":0,"I":0},"last_write_ts":{"T":0,"I":0},"hb":{"T":0,"I":0},"status":"error","conditions":null,"n":null,"error":"get data for restore: open filelist \"2026-07-17T00:58:29Z/cfg/filelist.pbm\": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/cfg/filelist.pbm: get OSS object header: operation error HeadObject: Head \"https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/cfg/filelist.pbm\": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout","pbm_version":"","balancer":""}} 2026-07-17T01:09:55.414Z INFO Restore state changed {"controller": "psmdbrestore-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBRestore", "PerconaServerMongoDBRestore": {"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"}, "namespace": "demand-backup-physical-sharded-oss-native-7616", "name": "restore-backup-oss-native-sharded", "reconcileID": "17dadb5f-c9c9-4570-8f56-4fd0a96f40e0", "previous": "requested", "current": "error"} + kubectl_bin get psmdb-restore restore-backup-oss-native-sharded -o yaml ++ mktemp + local LAST_OUT=/tmp/tmp.1a0AcV9jcq ++ mktemp + local LAST_ERR=/tmp/tmp.iqW42t2jVn + local exit_status=0 + local timeout=4 + args=('get' 'psmdb-restore' 'restore-backup-oss-native-sharded' '-o' 'yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get psmdb-restore restore-backup-oss-native-sharded -o yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1a0AcV9jcq apiVersion: psmdb.percona.com/v1 kind: PerconaServerMongoDBRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDBRestore","metadata":{"annotations":{},"name":"restore-backup-oss-native-sharded","namespace":"demand-backup-physical-sharded-oss-native-7616"},"spec":{"backupName":"backup-oss-native-sharded","clusterName":"some-name"}} creationTimestamp: "2026-07-17T01:01:51Z" generation: 1 name: restore-backup-oss-native-sharded namespace: demand-backup-physical-sharded-oss-native-7616 resourceVersion: "1784250595427311006" uid: fda40730-d22d-4ec2-85d0-8217f2a6df44 spec: backupName: backup-oss-native-sharded clusterName: some-name status: error: 'get data for restore: open filelist "2026-07-17T00:58:29Z/cfg/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/cfg/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/cfg/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout' pbmName: "2026-07-17T01:08:00.410100184Z" state: error + cat /tmp/tmp.iqW42t2jVn + rm /tmp/tmp.1a0AcV9jcq /tmp/tmp.iqW42t2jVn + return 0 + kubectl_bin describe psmdb-restore restore-backup-oss-native-sharded ++ mktemp + local LAST_OUT=/tmp/tmp.1lyccCkB7U ++ mktemp + local LAST_ERR=/tmp/tmp.Utnmqm6vY8 + local exit_status=0 + local timeout=4 + args=('describe' 'psmdb-restore' 'restore-backup-oss-native-sharded') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl describe psmdb-restore restore-backup-oss-native-sharded + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1lyccCkB7U Name: restore-backup-oss-native-sharded Namespace: demand-backup-physical-sharded-oss-native-7616 Labels: Annotations: API Version: psmdb.percona.com/v1 Kind: PerconaServerMongoDBRestore Metadata: Creation Timestamp: 2026-07-17T01:01:51Z Generation: 1 Resource Version: 1784250595427311006 UID: fda40730-d22d-4ec2-85d0-8217f2a6df44 Spec: Backup Name: backup-oss-native-sharded Cluster Name: some-name Status: Error: get data for restore: open filelist "2026-07-17T00:58:29Z/cfg/filelist.pbm": list with parts for mw source reader: fetching pbm file parts base for 2026-07-17T00:58:29Z/cfg/filelist.pbm: get OSS object header: operation error HeadObject: Head "https://operator-testing.oss-eu-central-1.aliyuncs.com/psmdb-demand-backup-physical-sharded-oss-native/2026-07-17T00%3A58%3A29Z/cfg/filelist.pbm": dial tcp: lookup operator-testing.oss-eu-central-1.aliyuncs.com: i/o timeout Pbm Name: 2026-07-17T01:08:00.410100184Z State: error Events: + cat /tmp/tmp.Utnmqm6vY8 + rm /tmp/tmp.1lyccCkB7U /tmp/tmp.Utnmqm6vY8 + return 0 + echo 'Restore object restore-backup-oss-native-sharded is in error state after 2 minutes.' Restore object restore-backup-oss-native-sharded is in error state after 2 minutes. + echo something went wrong with operator or kubernetes cluster something went wrong with operator or kubernetes cluster + exit 1 [01:10:19.651234] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. [01:10:20.681375] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. [01:10:22.119092] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. [01:10:23.076790] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. [01:10:24.080209] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. [01:10:25.132846] WARNING kubectl error: No resources found in monitoring-pmm3-14502 namespace. FAILED =================================== FAILURES =================================== ______________________________ test_bash_wrapper _______________________________ request = > def test_bash_wrapper(request: pytest.FixtureRequest) -> None: """Run a bash test script via pytest wrapper.""" test_name = request.config.getoption("--test-name") if not test_name: pytest.skip("No --test-name provided") > run_bash_test(test_name) e2e-tests/test_pytest_wrapper.py:10: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test_name = 'demand-backup-physical-sharded-oss-native' def run_bash_test(test_name: str) -> None: """Run bash script with stdout/stderr inherited from the pytest process.""" script_path = Path(__file__).parent.parent / test_name / "run" if not script_path.exists(): pytest.fail(f"Script not found: {script_path}") original_cwd = os.getcwd() script_dir = script_path.parent try: os.chdir(script_dir) result = subprocess.run(["bash", "run"]) if result.returncode != 0: > pytest.fail(f"Test {test_name} failed with exit code {result.returncode}") E Failed: Test demand-backup-physical-sharded-oss-native failed with exit code 1 e2e-tests/lib/bash_wrapper.py:27: Failed ------------------------------ Captured log setup ------------------------------ WARNING lib.kubectl:kubectl.py:18 kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 INFO conftest:conftest.py:182 Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operator:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mongodb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1-1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-dev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersync-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 - generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/demand-backup-physical-sharded-oss-native.xml - - Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/demand-backup-physical-sharded-oss-native.html - =========================== short test summary info ============================ FAILED e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[demand-backup-physical-sharded-oss-native] - Failed: Test demand-backup-physical-sharded-oss-native failed with exit code 1 ======================== 1 failed in 1319.71s (0:21:59) ========================