============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[demand-backup] [00:01:31.444535] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [00:01:34.051509] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + create_infra demand-backup-1939 + local ns=demand-backup-1939 + echo demand-backup-1939 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.2CHAHCckUH ++ mktemp + local LAST_ERR=/tmp/tmp.s5sN7xd4cI + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2CHAHCckUH + cat /tmp/tmp.s5sN7xd4cI + rm /tmp/tmp.2CHAHCckUH /tmp/tmp.s5sN7xd4cI + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.W1eZ8xBVyo ++ mktemp + local LAST_ERR=/tmp/tmp.oOnRvd6x9Y + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W1eZ8xBVyo + cat /tmp/tmp.oOnRvd6x9Y + rm /tmp/tmp.W1eZ8xBVyo /tmp/tmp.oOnRvd6x9Y + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.6bzvUS0nA8 ++ mktemp + local LAST_ERR=/tmp/tmp.1IvkskG50p + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6bzvUS0nA8 + cat /tmp/tmp.1IvkskG50p + rm /tmp/tmp.6bzvUS0nA8 /tmp/tmp.1IvkskG50p + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.KEqgj56qN5 ++ mktemp + local LAST_ERR=/tmp/tmp.lXOU35VI5w + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KEqgj56qN5 + cat /tmp/tmp.lXOU35VI5w + rm /tmp/tmp.KEqgj56qN5 /tmp/tmp.lXOU35VI5w + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.EBudsmIgRp ++ mktemp + local LAST_ERR=/tmp/tmp.I7sECnLzfI + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EBudsmIgRp + cat /tmp/tmp.I7sECnLzfI + rm /tmp/tmp.EBudsmIgRp /tmp/tmp.I7sECnLzfI + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.UdFrmyx2UI ++ mktemp + local LAST_ERR=/tmp/tmp.aIuF4MKp2C + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UdFrmyx2UI + cat /tmp/tmp.aIuF4MKp2C + rm /tmp/tmp.UdFrmyx2UI /tmp/tmp.aIuF4MKp2C + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8MkAR0tSpw +++ mktemp ++ local LAST_ERR=/tmp/tmp.QHZdFbu0gZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.8MkAR0tSpw ++ cat /tmp/tmp.QHZdFbu0gZ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.8MkAR0tSpw ++ cat /tmp/tmp.QHZdFbu0gZ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.8MkAR0tSpw ++ cat /tmp/tmp.QHZdFbu0gZ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.8MkAR0tSpw ++ cat /tmp/tmp.QHZdFbu0gZ Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.8MkAR0tSpw /tmp/tmp.QHZdFbu0gZ ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + [[ -n '' ]] + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.vGBCxQw9Ed ++ mktemp + awk '{print$1}' + local LAST_ERR=/tmp/tmp.iGDf3S8l2J + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + xargs kubectl delete ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.IGcYXNNoOk ++ mktemp + local LAST_ERR=/tmp/tmp.LGfpKefj6H + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vGBCxQw9Ed + cat /tmp/tmp.iGDf3S8l2J + rm /tmp/tmp.vGBCxQw9Ed /tmp/tmp.iGDf3S8l2J + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IGcYXNNoOk + cat /tmp/tmp.LGfpKefj6H + rm /tmp/tmp.IGcYXNNoOk /tmp/tmp.LGfpKefj6H + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.umdKcvyY8e ++ mktemp + local LAST_ERR=/tmp/tmp.1CszejKxiR + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.umdKcvyY8e + cat /tmp/tmp.1CszejKxiR + rm /tmp/tmp.umdKcvyY8e /tmp/tmp.1CszejKxiR + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.eVNm3BkGYg ++ mktemp + local LAST_ERR=/tmp/tmp.NqqXTI6Q3R + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eVNm3BkGYg namespace/psmdb-operator created + cat /tmp/tmp.NqqXTI6Q3R + rm /tmp/tmp.eVNm3BkGYg /tmp/tmp.NqqXTI6Q3R + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.DTPezX1DZ8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.pFuPSV46xU ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DTPezX1DZ8 ++ cat /tmp/tmp.pFuPSV46xU ++ rm /tmp/tmp.DTPezX1DZ8 /tmp/tmp.pFuPSV46xU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.oJwGOl0k2I ++ mktemp + local LAST_ERR=/tmp/tmp.lEQeZyW6Ae + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oJwGOl0k2I Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12" modified. + cat /tmp/tmp.lEQeZyW6Ae + rm /tmp/tmp.oJwGOl0k2I /tmp/tmp.lEQeZyW6Ae + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.SPYiKbI2fr ++ mktemp + local LAST_ERR=/tmp/tmp.XOvBhZgmKy + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SPYiKbI2fr customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.XOvBhZgmKy + rm /tmp/tmp.SPYiKbI2fr /tmp/tmp.XOvBhZgmKy + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.dXoo4eXPHC ++ mktemp + local LAST_ERR=/tmp/tmp.jbGUlkRNST + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dXoo4eXPHC clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.jbGUlkRNST + rm /tmp/tmp.dXoo4eXPHC /tmp/tmp.jbGUlkRNST + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.COTL4bzKI5 ++ mktemp + local LAST_ERR=/tmp/tmp.QAWF8QXDtM + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.COTL4bzKI5 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.QAWF8QXDtM + rm /tmp/tmp.COTL4bzKI5 /tmp/tmp.QAWF8QXDtM + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z9kwbf0Spu +++ mktemp ++ local LAST_ERR=/tmp/tmp.N4dT0QZRLB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Z9kwbf0Spu ++ cat /tmp/tmp.N4dT0QZRLB ++ rm /tmp/tmp.Z9kwbf0Spu /tmp/tmp.N4dT0QZRLB ++ return 0 + wait_operator_pod percona-server-mongodb-operator-66cd7b75cf-s8r6x + local pod=percona-server-mongodb-operator-66cd7b75cf-s8r6x + set +o xtrace waiting for pod/percona-server-mongodb-operator-66cd7b75cf-s8r6x to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.f2Kuh7F7OP +++ mktemp ++ local LAST_ERR=/tmp/tmp.s0WNj3SrYy ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f2Kuh7F7OP ++ cat /tmp/tmp.s0WNj3SrYy ++ rm /tmp/tmp.f2Kuh7F7OP /tmp/tmp.s0WNj3SrYy ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-s8r6x ++ mktemp + local LAST_OUT=/tmp/tmp.TVWkyYL3tn ++ mktemp + local LAST_ERR=/tmp/tmp.OIPQ6q8TBN + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-s8r6x') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-s8r6x + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TVWkyYL3tn + cat /tmp/tmp.OIPQ6q8TBN + rm /tmp/tmp.TVWkyYL3tn /tmp/tmp.OIPQ6q8TBN + return 0 2026-07-17T00:02:37.960Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-1939 + local namespace=demand-backup-1939 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + [[ -n '' ]] + desc 'cleaned up old namespaces demand-backup-1939' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-1939 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-1939 --ignore-not-found + awk '{print$1}' ++ mktemp + local LAST_OUT=/tmp/tmp.eEAjmwBbsR ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.S73TbK38RX + local LAST_ERR=/tmp/tmp.bH2e9OE4qz ++ mktemp + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + local LAST_ERR=/tmp/tmp.rjkTeaWk7a + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'demand-backup-1939' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-1939 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eEAjmwBbsR + cat /tmp/tmp.bH2e9OE4qz + rm /tmp/tmp.eEAjmwBbsR /tmp/tmp.bH2e9OE4qz + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S73TbK38RX + cat /tmp/tmp.rjkTeaWk7a + rm /tmp/tmp.S73TbK38RX /tmp/tmp.rjkTeaWk7a + return 0 + kubectl_bin wait --for=delete namespace demand-backup-1939 ++ mktemp + local LAST_OUT=/tmp/tmp.fTHHXNActu ++ mktemp + local LAST_ERR=/tmp/tmp.PD9pluBFwb + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'demand-backup-1939') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-1939 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fTHHXNActu + cat /tmp/tmp.PD9pluBFwb + rm /tmp/tmp.fTHHXNActu /tmp/tmp.PD9pluBFwb + return 0 + desc 'create namespace demand-backup-1939' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-1939 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-1939 ++ mktemp + local LAST_OUT=/tmp/tmp.TJd4S3flPQ ++ mktemp + local LAST_ERR=/tmp/tmp.Xx65zUKFsz + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'demand-backup-1939') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-1939 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TJd4S3flPQ namespace/demand-backup-1939 created + cat /tmp/tmp.Xx65zUKFsz + rm /tmp/tmp.TJd4S3flPQ /tmp/tmp.Xx65zUKFsz + return 0 + set_kube_ctx demand-backup-1939 + local namespace=demand-backup-1939 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.AAbs1POoC9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZcSz09LHy2 ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AAbs1POoC9 ++ cat /tmp/tmp.ZcSz09LHy2 ++ rm /tmp/tmp.AAbs1POoC9 /tmp/tmp.ZcSz09LHy2 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12 --namespace=demand-backup-1939 ++ mktemp + local LAST_OUT=/tmp/tmp.5mBC4O08mF ++ mktemp + local LAST_ERR=/tmp/tmp.V7tBvmYitN + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12' '--namespace=demand-backup-1939') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12 --namespace=demand-backup-1939 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5mBC4O08mF Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster12" modified. + cat /tmp/tmp.V7tBvmYitN + rm /tmp/tmp.5mBC4O08mF /tmp/tmp.V7tBvmYitN + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + arch_scheduling_enabled + [[ -n '' ]] + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Fri Jul 17 00:03:13 2026 NAMESPACE: demand-backup-1939 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-1939.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-1939 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-1939 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-1939 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-1939 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7X46Xo6POr +++ mktemp ++ local LAST_ERR=/tmp/tmp.QxcNqmc9iK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=release=minio-service' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7X46Xo6POr ++ cat /tmp/tmp.QxcNqmc9iK ++ rm /tmp/tmp.7X46Xo6POr /tmp/tmp.QxcNqmc9iK ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-92xzd + wait_pod minio-service-6d5f646cdc-92xzd + local pod=minio-service-6d5f646cdc-92xzd + set +o xtrace waiting for pod/minio-service-6d5f646cdc-92xzd to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-1939.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + aws_cli_minio http://minio-service:9000 s3 mb s3://operator-testing + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.s9QRnOEMvS ++ mktemp + local LAST_ERR=/tmp/tmp.TiwqLiU9IG + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'mb' 's3://operator-testing') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s9QRnOEMvS pod "aws-cli" deleted from demand-backup-1939 namespace + cat /tmp/tmp.TiwqLiU9IG All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.s9QRnOEMvS /tmp/tmp.TiwqLiU9IG + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + cluster=some-name-rs0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5DXaUNXl95 ++ mktemp + local LAST_ERR=/tmp/tmp.9t0JX2kP61 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5DXaUNXl95 secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.9t0JX2kP61 + rm /tmp/tmp.5DXaUNXl95 /tmp/tmp.9t0JX2kP61 + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.aYfkx9fM3r ++ mktemp + local LAST_ERR=/tmp/tmp.VNKEGHjsGg + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aYfkx9fM3r secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/oci-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.VNKEGHjsGg + rm /tmp/tmp.aYfkx9fM3r /tmp/tmp.VNKEGHjsGg + return 0 + desc 'create first PSMDB cluster some-name-rs0' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name-rs0 ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/some-name-rs0.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + local LAST_OUT=/tmp/tmp.vqyBlGsApM ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '.spec.upgradeOptions.apply="Never"' + add_arch_to_psmdb - + arch_scheduling_enabled + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-1939/g + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + local LAST_ERR=/tmp/tmp.VOeDB4CDDj + [[ -n '' ]] + cat - + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vqyBlGsApM perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.VOeDB4CDDj + rm /tmp/tmp.vqyBlGsApM /tmp/tmp.VOeDB4CDDj + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready..................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9kFYdI0yXb +++ mktemp ++ local LAST_ERR=/tmp/tmp.2vdTe6ovmW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9kFYdI0yXb ++ cat /tmp/tmp.2vdTe6ovmW ++ rm /tmp/tmp.9kFYdI0yXb /tmp/tmp.2vdTe6ovmW ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready................OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4zxj7AuQJj +++ mktemp ++ local LAST_ERR=/tmp/tmp.YMhDoMeBfX ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4zxj7AuQJj ++ cat /tmp/tmp.YMhDoMeBfX ++ rm /tmp/tmp.4zxj7AuQJj /tmp/tmp.YMhDoMeBfX ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YJaw5LsAPq +++ mktemp ++ local LAST_ERR=/tmp/tmp.U8t6gZxgIG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YJaw5LsAPq ++ cat /tmp/tmp.U8t6gZxgIG ++ rm /tmp/tmp.YJaw5LsAPq /tmp/tmp.U8t6gZxgIG ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready..... + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.owL0ZbzDsn/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-1939", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.PhpJCFXud2 ++ mktemp + local LAST_ERR=/tmp/tmp.YSxPcizGcj + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PhpJCFXud2 + cat /tmp/tmp.YSxPcizGcj + rm /tmp/tmp.PhpJCFXud2 /tmp/tmp.YSxPcizGcj + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.owL0ZbzDsn/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.owL0ZbzDsn/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.owL0ZbzDsn/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.owL0ZbzDsn/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-07-17T00:06:32+0000] compare_kubectl: statefulset/some-name-rs0 OK + check_custom_backup_probes some-name-rs0 + local cluster=some-name-rs0 + desc 'verify custom backup-agent liveness/readiness probes set via CR are applied' + set +o xtrace ----------------------------------------------------------------------------------- verify custom backup-agent liveness/readiness probes set via CR are applied ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.MI3xqgnNJY ++ mktemp + local LAST_ERR=/tmp/tmp.VHR3jZklVd + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[ {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}}, {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json $'-p=[\n\t {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MI3xqgnNJY perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.VHR3jZklVd + rm /tmp/tmp.MI3xqgnNJY /tmp/tmp.VHR3jZklVd + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fBtsxQkksC +++ mktemp ++ local LAST_ERR=/tmp/tmp.wYdjBfpufq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fBtsxQkksC ++ cat /tmp/tmp.wYdjBfpufq ++ rm /tmp/tmp.fBtsxQkksC /tmp/tmp.wYdjBfpufq ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jNphVT60ag +++ mktemp ++ local LAST_ERR=/tmp/tmp.43nAiijmDN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jNphVT60ag ++ cat /tmp/tmp.43nAiijmDN ++ rm /tmp/tmp.jNphVT60ag /tmp/tmp.43nAiijmDN ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vR8AdMTsvW +++ mktemp ++ local LAST_ERR=/tmp/tmp.eUTWGZKMk8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vR8AdMTsvW ++ cat /tmp/tmp.eUTWGZKMk8 ++ rm /tmp/tmp.vR8AdMTsvW /tmp/tmp.eUTWGZKMk8 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.......................................... + local liveness readiness ++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UpoD1DV7so +++ mktemp ++ local LAST_ERR=/tmp/tmp.aGsKEyJ5NR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UpoD1DV7so ++ cat /tmp/tmp.aGsKEyJ5NR ++ rm /tmp/tmp.UpoD1DV7so /tmp/tmp.aGsKEyJ5NR ++ return 0 + liveness=/bin/true ++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ASUj7jlQXt +++ mktemp ++ local LAST_ERR=/tmp/tmp.ja7XiO4OHF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ASUj7jlQXt ++ cat /tmp/tmp.ja7XiO4OHF ++ rm /tmp/tmp.ASUj7jlQXt /tmp/tmp.ja7XiO4OHF ++ return 0 + readiness=/bin/true + [[ /bin/true != /bin/true ]] + [[ /bin/true != /bin/true ]] + desc 'revert custom backup-agent probes to defaults' + set +o xtrace ----------------------------------------------------------------------------------- revert custom backup-agent probes to defaults ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.Rvlf1km9wC ++ mktemp + local LAST_ERR=/tmp/tmp.kyJjKWMYS5 + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[ {"op":"remove","path":"/spec/backup/livenessProbe"}, {"op":"remove","path":"/spec/backup/readinessProbe"} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json $'-p=[\n\t {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Rvlf1km9wC perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.kyJjKWMYS5 + rm /tmp/tmp.Rvlf1km9wC /tmp/tmp.kyJjKWMYS5 + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J3cJ2T3P5o +++ mktemp ++ local LAST_ERR=/tmp/tmp.mA3kAs89kD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J3cJ2T3P5o ++ cat /tmp/tmp.mA3kAs89kD ++ rm /tmp/tmp.J3cJ2T3P5o /tmp/tmp.mA3kAs89kD ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8QYFPuQ5g6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tERhpG5nOS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8QYFPuQ5g6 ++ cat /tmp/tmp.tERhpG5nOS ++ rm /tmp/tmp.8QYFPuQ5g6 /tmp/tmp.tERhpG5nOS ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jwJhk5YSDr +++ mktemp ++ local LAST_ERR=/tmp/tmp.JfZpWcCbj1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jwJhk5YSDr ++ cat /tmp/tmp.JfZpWcCbj1 ++ rm /tmp/tmp.jwJhk5YSDr /tmp/tmp.JfZpWcCbj1 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready....................................... + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i8RmyZTlpj +++ mktemp ++ local LAST_ERR=/tmp/tmp.opNKDzZwvs ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.i8RmyZTlpj ++ cat /tmp/tmp.opNKDzZwvs ++ rm /tmp/tmp.i8RmyZTlpj /tmp/tmp.opNKDzZwvs ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.uHBOlZ59Bu ++ mktemp + local LAST_ERR=/tmp/tmp.p9Hg17cLra + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uHBOlZ59Bu Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("08686598-a938-40c5-89c5-0f66d7225382") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.p9Hg17cLra + rm /tmp/tmp.uHBOlZ59Bu /tmp/tmp.p9Hg17cLra + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yZZtNd2SlW +++ mktemp ++ local LAST_ERR=/tmp/tmp.cyk0pqElNP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yZZtNd2SlW ++ cat /tmp/tmp.cyk0pqElNP ++ rm /tmp/tmp.yZZtNd2SlW /tmp/tmp.cyk0pqElNP ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.BDhRRhpCY9 ++ mktemp + local LAST_ERR=/tmp/tmp.yiBg5w1i4o + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BDhRRhpCY9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b457a21e-c246-4606-bbf3-ca97acd90ee9") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.yiBg5w1i4o + rm /tmp/tmp.BDhRRhpCY9 /tmp/tmp.yiBg5w1i4o + return 0 + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NrnGdOGyIi +++ mktemp ++ local LAST_ERR=/tmp/tmp.oIWpzcpD57 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NrnGdOGyIi ++ cat /tmp/tmp.oIWpzcpD57 ++ rm /tmp/tmp.NrnGdOGyIi /tmp/tmp.oIWpzcpD57 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.sKd7kkros2 ++ mktemp + local LAST_ERR=/tmp/tmp.nsl04rIEfn + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sKd7kkros2 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1370e1eb-f211-428b-b308-e516a9d7a67d") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.nsl04rIEfn + rm /tmp/tmp.sKd7kkros2 /tmp/tmp.nsl04rIEfn + return 0 + run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test3.insert({ x: 100502 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cHzYutBjDQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5KM8b02H4s ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cHzYutBjDQ ++ cat /tmp/tmp.5KM8b02H4s ++ rm /tmp/tmp.cHzYutBjDQ /tmp/tmp.5KM8b02H4s ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.4kMcrF7YAq ++ mktemp + local LAST_ERR=/tmp/tmp.Ke7V5Ixv3x + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4kMcrF7YAq Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1342787a-f6cb-4776-b42f-da3618ea3a3b") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Ke7V5Ixv3x + rm /tmp/tmp.4kMcrF7YAq /tmp/tmp.Ke7V5Ixv3x + return 0 + custom_user_name=test1user + custom_role_name=test1role + run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vF9Y7MYdwW +++ mktemp ++ local LAST_ERR=/tmp/tmp.2m0PNiCAv1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vF9Y7MYdwW ++ cat /tmp/tmp.2m0PNiCAv1 ++ rm /tmp/tmp.vF9Y7MYdwW /tmp/tmp.2m0PNiCAv1 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.U4ECPClYqr ++ mktemp + local LAST_ERR=/tmp/tmp.JyIu2yYylB + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U4ECPClYqr Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("cb9bb88d-44b5-4df8-b22b-9e366ef82cf2") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp Successfully added user: { "user" : "test1user", "roles" : [ ] } bye + cat /tmp/tmp.JyIu2yYylB + rm /tmp/tmp.U4ECPClYqr /tmp/tmp.JyIu2yYylB + return 0 + run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IZ3almgIqg +++ mktemp ++ local LAST_ERR=/tmp/tmp.pLaqbDICMf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IZ3almgIqg ++ cat /tmp/tmp.pLaqbDICMf ++ rm /tmp/tmp.IZ3almgIqg /tmp/tmp.pLaqbDICMf ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.PvHrvfn6X9 ++ mktemp + local LAST_ERR=/tmp/tmp.e3ANa9bSPa + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PvHrvfn6X9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c079a186-cb85-49ba-869b-175791f55be4") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp { "role" : "test1role", "privileges" : [ ], "roles" : [ ] } bye + cat /tmp/tmp.e3ANa9bSPa + rm /tmp/tmp.PvHrvfn6X9 /tmp/tmp.e3ANa9bSPa + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:10:19+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q2zckjZrln +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qsj8HMxkQF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q2zckjZrln ++ cat /tmp/tmp.Qsj8HMxkQF ++ rm /tmp/tmp.q2zckjZrln /tmp/tmp.Qsj8HMxkQF ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.1VelYvEQ7o ++ mktemp + local LAST_ERR=/tmp/tmp.Y64l8t6q3h + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1VelYvEQ7o + cat /tmp/tmp.Y64l8t6q3h + rm /tmp/tmp.1VelYvEQ7o /tmp/tmp.Y64l8t6q3h + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:10:22+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2teNSBnU0B +++ mktemp ++ local LAST_ERR=/tmp/tmp.SOOhAayjGS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2teNSBnU0B ++ cat /tmp/tmp.SOOhAayjGS ++ rm /tmp/tmp.2teNSBnU0B /tmp/tmp.SOOhAayjGS ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.mh1ihI3g4l ++ mktemp + local LAST_ERR=/tmp/tmp.fpOSi0MbgZ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mh1ihI3g4l + cat /tmp/tmp.fpOSi0MbgZ + rm /tmp/tmp.mh1ihI3g4l /tmp/tmp.fpOSi0MbgZ + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:10:24+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r2s9rjaPRW +++ mktemp ++ local LAST_ERR=/tmp/tmp.XxmykzFtrE ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r2s9rjaPRW ++ cat /tmp/tmp.XxmykzFtrE ++ rm /tmp/tmp.r2s9rjaPRW /tmp/tmp.XxmykzFtrE ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.EDaHFb6PQY ++ mktemp + local LAST_ERR=/tmp/tmp.I3Z9Gp76iP + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EDaHFb6PQY + cat /tmp/tmp.I3Z9Gp76iP + rm /tmp/tmp.EDaHFb6PQY /tmp/tmp.I3Z9Gp76iP + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-07-17T00:09:18.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-07-17T00:08:47.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-07-17T00:09:47.000+0000 I listening for the commands + '[' -z '' ']' + desc 'Check GCS profiles' + set +o xtrace ----------------------------------------------------------------------------------- Check GCS profiles ----------------------------------------------------------------------------------- + compare_pbm_profile_setup some-name gcp-cs-s3 + local cluster=some-name + local profile=gcp-cs-s3 + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3 ++ mktemp + local LAST_OUT=/tmp/tmp.wQyoJod2cu ++ mktemp + local LAST_ERR=/tmp/tmp.GSBDqAHZct + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-s3') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wQyoJod2cu + cat /tmp/tmp.GSBDqAHZct + rm /tmp/tmp.wQyoJod2cu /tmp/tmp.GSBDqAHZct + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.owL0ZbzDsn/pbm_profile_gcp-cs-s3.yml + compare_pbm_profile_setup some-name gcp-cs-sa + local cluster=some-name + local profile=gcp-cs-sa + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa ++ mktemp + local LAST_OUT=/tmp/tmp.htWG0uGlgb ++ mktemp + local LAST_ERR=/tmp/tmp.o8VFOZpW9t + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-sa') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.htWG0uGlgb + cat /tmp/tmp.o8VFOZpW9t + rm /tmp/tmp.htWG0uGlgb /tmp/tmp.o8VFOZpW9t + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.owL0ZbzDsn/pbm_profile_gcp-cs-sa.yml + backup_name_minio=backup-minio + '[' -z '' ']' + backup_name_aws=backup-aws-s3 + backup_name_gcp_s3=backup-gcp-cs-s3 + backup_name_gcp_sa=backup-gcp-cs-sa + backup_name_azure=backup-azure-blob + backup_name_oss_native=backup-oss-native + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-07-17T00:10:34+0000] running backup backup-aws-s3 + yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.QqQEmNwUCL ++ mktemp + local LAST_ERR=/tmp/tmp.hrZQ8wuVUG + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QqQEmNwUCL perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.hrZQ8wuVUG + rm /tmp/tmp.QqQEmNwUCL /tmp/tmp.hrZQ8wuVUG + return 0 + run_backup gcp-cs-s3 + local storage=gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local type=logical + log 'running backup backup-gcp-cs-s3' + set +o xtrace [2026-07-17T00:10:36+0000] running backup backup-gcp-cs-s3 + yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.I6szU8aYrC ++ mktemp + local LAST_ERR=/tmp/tmp.dmWGqbMxJh + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I6szU8aYrC perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created + cat /tmp/tmp.dmWGqbMxJh + rm /tmp/tmp.I6szU8aYrC /tmp/tmp.dmWGqbMxJh + return 0 + run_backup gcp-cs-sa + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=logical + log 'running backup backup-gcp-cs-sa' + set +o xtrace [2026-07-17T00:10:39+0000] running backup backup-gcp-cs-sa + yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Bibf78L82f ++ mktemp + local LAST_ERR=/tmp/tmp.wcHpo5l0aB + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bibf78L82f perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.wcHpo5l0aB + rm /tmp/tmp.Bibf78L82f /tmp/tmp.wcHpo5l0aB + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-07-17T00:10:41+0000] running backup backup-azure-blob + yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.maP0dLvVO3 ++ mktemp + local LAST_ERR=/tmp/tmp.BYXs5IYfOR + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.maP0dLvVO3 perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.BYXs5IYfOR + rm /tmp/tmp.maP0dLvVO3 /tmp/tmp.BYXs5IYfOR + return 0 + run_backup oss-native + local storage=oss-native + local backup_name=backup-oss-native + local type=logical + log 'running backup backup-oss-native' + set +o xtrace [2026-07-17T00:10:44+0000] running backup backup-oss-native + yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.mb9Wpez15j ++ mktemp + local LAST_ERR=/tmp/tmp.mb5gERSPZg + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mb9Wpez15j perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created + cat /tmp/tmp.mb5gERSPZg + rm /tmp/tmp.mb9Wpez15j /tmp/tmp.mb5gERSPZg + return 0 + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-07-17T00:10:46+0000] running backup backup-minio + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.SUpScvfWrt ++ mktemp + local LAST_ERR=/tmp/tmp.p0InLGXu66 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SUpScvfWrt perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.p0InLGXu66 + rm /tmp/tmp.SUpScvfWrt /tmp/tmp.p0InLGXu66 + return 0 + '[' -z '' ']' + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state..OK + wait_backup backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-s3 to reach ready state..............................OK + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state.......OK + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state..OK + wait_backup backup-oss-native + local backup_name=backup-oss-native + local target_state=ready + set +o xtrace waiting for backup-oss-native to reach ready state..OK + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state..OK + sleep 5 + '[' -z '' ']' + desc 'check backup and restore -- aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- aws-s3 ----------------------------------------------------------------------------------- ++ get_backup_dest backup-aws-s3 ++ local backup_name=backup-aws-s3 ++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.QcaWwkMH41 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nwmdMWSOvF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QcaWwkMH41 ++ cat /tmp/tmp.nwmdMWSOvF ++ rm /tmp/tmp.QcaWwkMH41 /tmp/tmp.nwmdMWSOvF ++ return 0 + backup_dest_aws=operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z + setup_aws_credentials + local secret_name=aws-s3-secret + [[ -n AKIARXP3OARBMEQ6FSEW ]] + [[ -n gljUukL3/TuQnzrtJUgiURH6vqjt+SEo89jGS2Z/ ]] + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-07-17T00:10:37Z + key=/rs0/myApp.test.gz + endpoint= + storage_name=aws-s3 + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n '' ]] + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-07-17T00:10:37Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z//rs0/myApp.test.gz found in aws-s3' Backup operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z//rs0/myApp.test.gz found in aws-s3 + run_recovery_check backup-aws-s3 some-name-rs0 + local backup=backup-aws-s3 + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HpdMeQ0vhh +++ mktemp ++ local LAST_ERR=/tmp/tmp.DGKozfMtVQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HpdMeQ0vhh ++ cat /tmp/tmp.DGKozfMtVQ ++ rm /tmp/tmp.HpdMeQ0vhh /tmp/tmp.DGKozfMtVQ ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.N1oFUXV9x2 ++ mktemp + local LAST_ERR=/tmp/tmp.ew4kSumQ8A + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.N1oFUXV9x2 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f1ce61b5-1263-4242-ba95-022c13963817") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ew4kSumQ8A + rm /tmp/tmp.N1oFUXV9x2 /tmp/tmp.ew4kSumQ8A + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:12:16+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5vPBfGwG6C +++ mktemp ++ local LAST_ERR=/tmp/tmp.BvMNJaOeBT ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5vPBfGwG6C ++ cat /tmp/tmp.BvMNJaOeBT ++ rm /tmp/tmp.5vPBfGwG6C /tmp/tmp.BvMNJaOeBT ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.9gZSQOQGs1 ++ mktemp + local LAST_ERR=/tmp/tmp.LHdL0aoERu + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9gZSQOQGs1 + cat /tmp/tmp.LHdL0aoERu + rm /tmp/tmp.9gZSQOQGs1 /tmp/tmp.LHdL0aoERu + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-aws-s3 + local backup_name=backup-aws-s3 + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-aws-s3' + set +o xtrace [2026-07-17T00:12:19+0000] running restore restore-backup-aws-s3 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/' + /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.bSil6CG9bT ++ mktemp + local LAST_ERR=/tmp/tmp.J7FZUs2TKr + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bSil6CG9bT perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created + cat /tmp/tmp.J7FZUs2TKr + rm /tmp/tmp.bSil6CG9bT /tmp/tmp.J7FZUs2TKr + return 0 + wait_restore backup-aws-s3 some-name + local backup_name=backup-aws-s3 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rV6QCMXUX9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.O5m185T0ID ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rV6QCMXUX9 ++ cat /tmp/tmp.O5m185T0ID ++ rm /tmp/tmp.rV6QCMXUX9 /tmp/tmp.O5m185T0ID ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:12:47+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.msf6wKs9eg +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZoWX2MYy0o ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.msf6wKs9eg ++ cat /tmp/tmp.ZoWX2MYy0o ++ rm /tmp/tmp.msf6wKs9eg /tmp/tmp.ZoWX2MYy0o ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SSclvfotRX ++ mktemp + local LAST_ERR=/tmp/tmp.nbEBRiRmFX + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SSclvfotRX + cat /tmp/tmp.nbEBRiRmFX + rm /tmp/tmp.SSclvfotRX /tmp/tmp.nbEBRiRmFX + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:12:49+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3S35I8eYjM +++ mktemp ++ local LAST_ERR=/tmp/tmp.GEEVGk7pwd ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3S35I8eYjM ++ cat /tmp/tmp.GEEVGk7pwd ++ rm /tmp/tmp.3S35I8eYjM /tmp/tmp.GEEVGk7pwd ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.4kC79tt54K ++ mktemp + local LAST_ERR=/tmp/tmp.W6hLVdIX6C + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4kC79tt54K + cat /tmp/tmp.W6hLVdIX6C + rm /tmp/tmp.4kC79tt54K /tmp/tmp.W6hLVdIX6C + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:12:52+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0DtE5o6Emp +++ mktemp ++ local LAST_ERR=/tmp/tmp.JeyxY1f4YH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0DtE5o6Emp ++ cat /tmp/tmp.JeyxY1f4YH ++ rm /tmp/tmp.0DtE5o6Emp /tmp/tmp.JeyxY1f4YH ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.7DTVlUW1Ou ++ mktemp + local LAST_ERR=/tmp/tmp.k7DKZXXdqk + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7DTVlUW1Ou + cat /tmp/tmp.k7DKZXXdqk + rm /tmp/tmp.7DTVlUW1Ou /tmp/tmp.k7DKZXXdqk + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'check backup and restore -- gcp-cs-s3' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs-s3 ----------------------------------------------------------------------------------- ++ get_backup_dest backup-gcp-cs-s3 ++ local backup_name=backup-gcp-cs-s3 ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ sed 's|oss://||' +++ mktemp ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.cYiiFENKp6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yyXjGusVB5 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cYiiFENKp6 ++ cat /tmp/tmp.yyXjGusVB5 ++ rm /tmp/tmp.cYiiFENKp6 /tmp/tmp.yyXjGusVB5 ++ return 0 + backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz found in gcp-cs + run_recovery_check backup-gcp-cs-s3 some-name-rs0 + local backup=backup-gcp-cs-s3 + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.piEVfcnv1d +++ mktemp ++ local LAST_ERR=/tmp/tmp.rwFrmpFnI5 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.piEVfcnv1d ++ cat /tmp/tmp.rwFrmpFnI5 ++ rm /tmp/tmp.piEVfcnv1d /tmp/tmp.rwFrmpFnI5 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.bkgmIk9H5z ++ mktemp + local LAST_ERR=/tmp/tmp.Bx7NoZFHCg + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bkgmIk9H5z Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("454b56e8-df00-4ab9-b318-b98f0e9bae48") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Bx7NoZFHCg + rm /tmp/tmp.bkgmIk9H5z /tmp/tmp.Bx7NoZFHCg + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:13:03+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aZW8wH3eZ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LH11fG23cl ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aZW8wH3eZ2 ++ cat /tmp/tmp.LH11fG23cl ++ rm /tmp/tmp.aZW8wH3eZ2 /tmp/tmp.LH11fG23cl ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.atOFohceUv ++ mktemp + local LAST_ERR=/tmp/tmp.VPr8pSJQb1 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.atOFohceUv + cat /tmp/tmp.VPr8pSJQb1 + rm /tmp/tmp.atOFohceUv /tmp/tmp.VPr8pSJQb1 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-gcp-cs-s3' + set +o xtrace [2026-07-17T00:13:05+0000] running restore restore-backup-gcp-cs-s3 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AaesCjtdxd ++ mktemp + local LAST_ERR=/tmp/tmp.sfipoltTuA + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AaesCjtdxd perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created + cat /tmp/tmp.sfipoltTuA + rm /tmp/tmp.AaesCjtdxd /tmp/tmp.sfipoltTuA + return 0 + wait_restore backup-gcp-cs-s3 some-name + local backup_name=backup-gcp-cs-s3 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z5oRfzZFKa +++ mktemp ++ local LAST_ERR=/tmp/tmp.ns7lf6u43T ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Z5oRfzZFKa ++ cat /tmp/tmp.ns7lf6u43T ++ rm /tmp/tmp.Z5oRfzZFKa /tmp/tmp.ns7lf6u43T ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:13:38+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rldZs54gcy +++ mktemp ++ local LAST_ERR=/tmp/tmp.2PKjyXDI8e ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rldZs54gcy ++ cat /tmp/tmp.2PKjyXDI8e ++ rm /tmp/tmp.rldZs54gcy /tmp/tmp.2PKjyXDI8e ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.aETTD1q6A4 ++ mktemp + local LAST_ERR=/tmp/tmp.m971tpVeUw + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aETTD1q6A4 + cat /tmp/tmp.m971tpVeUw + rm /tmp/tmp.aETTD1q6A4 /tmp/tmp.m971tpVeUw + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:13:41+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EpFV66Yh43 +++ mktemp ++ local LAST_ERR=/tmp/tmp.1xBwlGrSrn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EpFV66Yh43 ++ cat /tmp/tmp.1xBwlGrSrn ++ rm /tmp/tmp.EpFV66Yh43 /tmp/tmp.1xBwlGrSrn ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.lrEHq7ec1p ++ mktemp + local LAST_ERR=/tmp/tmp.tbBuRIps8b + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lrEHq7ec1p + cat /tmp/tmp.tbBuRIps8b + rm /tmp/tmp.lrEHq7ec1p /tmp/tmp.tbBuRIps8b + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:13:43+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0jdJjg7MTw +++ mktemp ++ local LAST_ERR=/tmp/tmp.bk0AbphBHi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0jdJjg7MTw ++ cat /tmp/tmp.bk0AbphBHi ++ rm /tmp/tmp.0jdJjg7MTw /tmp/tmp.bk0AbphBHi ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.FKLTZubmjY ++ mktemp + local LAST_ERR=/tmp/tmp.BtPVNqOCZr + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FKLTZubmjY + cat /tmp/tmp.BtPVNqOCZr + rm /tmp/tmp.FKLTZubmjY /tmp/tmp.BtPVNqOCZr + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'check backup and restore -- gcp-cs-sa' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs-sa ----------------------------------------------------------------------------------- ++ get_backup_dest backup-gcp-cs-sa ++ local backup_name=backup-gcp-cs-sa ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.5GU1AOXLge +++ mktemp ++ local LAST_ERR=/tmp/tmp.BbNlh0iM8i ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-sa' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ sed 's|oss://||' ++ sed 's|oci://||' +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5GU1AOXLge ++ cat /tmp/tmp.BbNlh0iM8i ++ rm /tmp/tmp.5GU1AOXLge /tmp/tmp.BbNlh0iM8i ++ return 0 + backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z + check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz found in gcp-cs + run_recovery_check backup-gcp-cs-sa some-name-rs0 + local backup=backup-gcp-cs-sa + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XCunuE2QCU +++ mktemp ++ local LAST_ERR=/tmp/tmp.TZEPuIQboZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XCunuE2QCU ++ cat /tmp/tmp.TZEPuIQboZ ++ rm /tmp/tmp.XCunuE2QCU /tmp/tmp.TZEPuIQboZ ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.DO4egKxnoA ++ mktemp + local LAST_ERR=/tmp/tmp.OmCJ7zyHz5 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DO4egKxnoA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8ee92ff6-e42e-46a3-aa2f-9baf84c73097") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.OmCJ7zyHz5 + rm /tmp/tmp.DO4egKxnoA /tmp/tmp.OmCJ7zyHz5 + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:13:51+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7ndHJY5Bcf +++ mktemp ++ local LAST_ERR=/tmp/tmp.uHG8zXpz0t ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7ndHJY5Bcf ++ cat /tmp/tmp.uHG8zXpz0t ++ rm /tmp/tmp.7ndHJY5Bcf /tmp/tmp.uHG8zXpz0t ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.XkNzZhD5ny ++ mktemp + local LAST_ERR=/tmp/tmp.jTZlAFDD2t + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XkNzZhD5ny + cat /tmp/tmp.jTZlAFDD2t + rm /tmp/tmp.XkNzZhD5ny /tmp/tmp.jTZlAFDD2t + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-gcp-cs-sa' + set +o xtrace [2026-07-17T00:13:53+0000] running restore restore-backup-gcp-cs-sa + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.J8VgRPD1xr ++ mktemp + local LAST_ERR=/tmp/tmp.4mosbQFmuo + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J8VgRPD1xr perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created + cat /tmp/tmp.4mosbQFmuo + rm /tmp/tmp.J8VgRPD1xr /tmp/tmp.4mosbQFmuo + return 0 + wait_restore backup-gcp-cs-sa some-name + local backup_name=backup-gcp-cs-sa + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JbVIUVBtvT +++ mktemp ++ local LAST_ERR=/tmp/tmp.DhlE7515dw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JbVIUVBtvT ++ cat /tmp/tmp.DhlE7515dw ++ rm /tmp/tmp.JbVIUVBtvT /tmp/tmp.DhlE7515dw ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:14:21+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0stKr3oF6m +++ mktemp ++ local LAST_ERR=/tmp/tmp.0yF6SFfYzZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0stKr3oF6m ++ cat /tmp/tmp.0yF6SFfYzZ ++ rm /tmp/tmp.0stKr3oF6m /tmp/tmp.0yF6SFfYzZ ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.0X5StRyHzT ++ mktemp + local LAST_ERR=/tmp/tmp.OLUvpoFQJk + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0X5StRyHzT + cat /tmp/tmp.OLUvpoFQJk + rm /tmp/tmp.0X5StRyHzT /tmp/tmp.OLUvpoFQJk + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:14:23+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TDre0mzbNR +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZXjMAbwABo ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TDre0mzbNR ++ cat /tmp/tmp.ZXjMAbwABo ++ rm /tmp/tmp.TDre0mzbNR /tmp/tmp.ZXjMAbwABo ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.FQqv7Zv4Xj ++ mktemp + local LAST_ERR=/tmp/tmp.TC825bufED + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FQqv7Zv4Xj + cat /tmp/tmp.TC825bufED + rm /tmp/tmp.FQqv7Zv4Xj /tmp/tmp.TC825bufED + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:14:25+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5g7pDDB7Ec +++ mktemp ++ local LAST_ERR=/tmp/tmp.zcavKFVTvo ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5g7pDDB7Ec ++ cat /tmp/tmp.zcavKFVTvo ++ rm /tmp/tmp.5g7pDDB7Ec /tmp/tmp.zcavKFVTvo ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.HiOCBUWq9W ++ mktemp + local LAST_ERR=/tmp/tmp.9geEUDhDU7 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HiOCBUWq9W + cat /tmp/tmp.9geEUDhDU7 + rm /tmp/tmp.HiOCBUWq9W /tmp/tmp.9geEUDhDU7 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'check backup and restore -- azure-blob' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- azure-blob ----------------------------------------------------------------------------------- ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' +++ mktemp ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ local LAST_OUT=/tmp/tmp.iBHGW5aJvJ +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|azure://||' ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.t4Zq262Wmb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iBHGW5aJvJ ++ cat /tmp/tmp.t4Zq262Wmb ++ rm /tmp/tmp.iBHGW5aJvJ /tmp/tmp.t4Zq262Wmb ++ return 0 + backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z + setup_azure_credentials + local secret_name=azure-secret + echo 'Setting up Azure credentials from K8s secret: azure-secret' Setting up Azure credentials from K8s secret: azure-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'Azure credentials configured successfully' Azure credentials configured successfully + check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z /rs0/myApp.test.gz ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup/2026-07-17T00:10:59Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz found in azure-blob' Backup psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz found in azure-blob + run_recovery_check backup-azure-blob some-name-rs0 + local backup=backup-azure-blob + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bcs1W3GuFd +++ mktemp ++ local LAST_ERR=/tmp/tmp.7KzhsM8KXn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bcs1W3GuFd ++ cat /tmp/tmp.7KzhsM8KXn ++ rm /tmp/tmp.bcs1W3GuFd /tmp/tmp.7KzhsM8KXn ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kTZ87Z6xjP ++ mktemp + local LAST_ERR=/tmp/tmp.Qbh7itm35B + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kTZ87Z6xjP Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7fdc19a6-b6fd-42b4-af01-1adff40875ac") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Qbh7itm35B + rm /tmp/tmp.kTZ87Z6xjP /tmp/tmp.Qbh7itm35B + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:14:35+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CYqzzVYJtX +++ mktemp ++ local LAST_ERR=/tmp/tmp.phyDTQMjlK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CYqzzVYJtX ++ cat /tmp/tmp.phyDTQMjlK ++ rm /tmp/tmp.CYqzzVYJtX /tmp/tmp.phyDTQMjlK ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.wjtzemQqtl ++ mktemp + local LAST_ERR=/tmp/tmp.DrkLBAKw2P + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wjtzemQqtl + cat /tmp/tmp.DrkLBAKw2P + rm /tmp/tmp.wjtzemQqtl /tmp/tmp.DrkLBAKw2P + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-azure-blob + local backup_name=backup-azure-blob + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-azure-blob' + set +o xtrace [2026-07-17T00:14:38+0000] running restore restore-backup-azure-blob + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.4tRMlmmaV4 ++ mktemp + local LAST_ERR=/tmp/tmp.M7qUAas6Lk + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4tRMlmmaV4 perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created + cat /tmp/tmp.M7qUAas6Lk + rm /tmp/tmp.4tRMlmmaV4 /tmp/tmp.M7qUAas6Lk + return 0 + wait_restore backup-azure-blob some-name + local backup_name=backup-azure-blob + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cyUyLck1QX +++ mktemp ++ local LAST_ERR=/tmp/tmp.dY8slNWr4X ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cyUyLck1QX ++ cat /tmp/tmp.dY8slNWr4X ++ rm /tmp/tmp.cyUyLck1QX /tmp/tmp.dY8slNWr4X ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:15:05+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1qnEALIjNp +++ mktemp ++ local LAST_ERR=/tmp/tmp.94plLP1yAW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1qnEALIjNp ++ cat /tmp/tmp.94plLP1yAW ++ rm /tmp/tmp.1qnEALIjNp /tmp/tmp.94plLP1yAW ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.3pbgolzUeb ++ mktemp + local LAST_ERR=/tmp/tmp.P34LA9Llml + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3pbgolzUeb + cat /tmp/tmp.P34LA9Llml + rm /tmp/tmp.3pbgolzUeb /tmp/tmp.P34LA9Llml + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:15:08+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0H72sJa4o9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8ZSGrTxCVg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0H72sJa4o9 ++ cat /tmp/tmp.8ZSGrTxCVg ++ rm /tmp/tmp.0H72sJa4o9 /tmp/tmp.8ZSGrTxCVg ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.mB71xNQVyY ++ mktemp + local LAST_ERR=/tmp/tmp.9Hs3Scvlds + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mB71xNQVyY + cat /tmp/tmp.9Hs3Scvlds + rm /tmp/tmp.mB71xNQVyY /tmp/tmp.9Hs3Scvlds + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:15:11+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dWCd67k9FE +++ mktemp ++ local LAST_ERR=/tmp/tmp.r056wZA0mk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dWCd67k9FE ++ cat /tmp/tmp.r056wZA0mk ++ rm /tmp/tmp.dWCd67k9FE /tmp/tmp.r056wZA0mk ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.HmgLXTeD8Y ++ mktemp + local LAST_ERR=/tmp/tmp.qsdENnwLp1 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HmgLXTeD8Y + cat /tmp/tmp.qsdENnwLp1 + rm /tmp/tmp.HmgLXTeD8Y /tmp/tmp.qsdENnwLp1 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'check backup and restore -- oss-native' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- oss-native ----------------------------------------------------------------------------------- ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' +++ mktemp ++ sed 's|azure://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.oEG4G2QUZV +++ mktemp ++ local LAST_ERR=/tmp/tmp.8lyXKiR4KK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oEG4G2QUZV ++ cat /tmp/tmp.8lyXKiR4KK ++ rm /tmp/tmp.oEG4G2QUZV /tmp/tmp.8lyXKiR4KK ++ return 0 + backup_dest_oss_native=operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9wH5jg14n5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dVbQFLSO1t +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9wH5jg14n5 +++ cat /tmp/tmp.dVbQFLSO1t +++ rm /tmp/tmp.9wH5jg14n5 /tmp/tmp.dVbQFLSO1t +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GlgGYDtd8m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Kx12eQ8dlV +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GlgGYDtd8m +++ cat /tmp/tmp.Kx12eQ8dlV +++ rm /tmp/tmp.GlgGYDtd8m /tmp/tmp.Kx12eQ8dlV +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bB7jHaHMuU +++ mktemp ++ local LAST_ERR=/tmp/tmp.5ATNsDKy8A ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bB7jHaHMuU ++ cat /tmp/tmp.5ATNsDKy8A ++ rm /tmp/tmp.bB7jHaHMuU /tmp/tmp.5ATNsDKy8A ++ return 0 + backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.r947qDmDjW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.k2yjt39W30 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.r947qDmDjW +++ cat /tmp/tmp.k2yjt39W30 +++ rm /tmp/tmp.r947qDmDjW /tmp/tmp.k2yjt39W30 +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KsNN8rirzU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.b9VnUOx6Yp +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KsNN8rirzU +++ cat /tmp/tmp.b9VnUOx6Yp +++ rm /tmp/tmp.KsNN8rirzU /tmp/tmp.b9VnUOx6Yp +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8KiMUrkSQ0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dRFEondATk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8KiMUrkSQ0 ++ cat /tmp/tmp.dRFEondATk ++ rm /tmp/tmp.8KiMUrkSQ0 /tmp/tmp.dRFEondATk ++ return 0 + backup_region_oss_native=eu-central-1 + check_backup_in_storage backup-oss-native oss rs0 myApp.test.gz + local backup=backup-oss-native + local storage_type=oss + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ sed -e 's/.json$//' ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oci://||' ++ sed 's|oss://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bmnXxjmpRg +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZQNz2xwPOW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bmnXxjmpRg ++ cat /tmp/tmp.ZQNz2xwPOW ++ rm /tmp/tmp.bmnXxjmpRg /tmp/tmp.ZQNz2xwPOW ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2zlIBMqUW2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HaRU9LhvQ7 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2zlIBMqUW2 +++ cat /tmp/tmp.HaRU9LhvQ7 +++ rm /tmp/tmp.2zlIBMqUW2 /tmp/tmp.HaRU9LhvQ7 +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lQMEd1Q4PL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TJhRrELP6S +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lQMEd1Q4PL +++ cat /tmp/tmp.TJhRrELP6S +++ rm /tmp/tmp.lQMEd1Q4PL /tmp/tmp.TJhRrELP6S +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LF0UNBQVmS +++ mktemp ++ local LAST_ERR=/tmp/tmp.EvKqMqOKtX ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LF0UNBQVmS ++ cat /tmp/tmp.EvKqMqOKtX ++ rm /tmp/tmp.LF0UNBQVmS /tmp/tmp.EvKqMqOKtX ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.T97xYBgwDB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MSJjjBKunS +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.T97xYBgwDB +++ cat /tmp/tmp.MSJjjBKunS +++ rm /tmp/tmp.T97xYBgwDB /tmp/tmp.MSJjjBKunS +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FXKTBnSYDA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JIkgazmGNg +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FXKTBnSYDA +++ cat /tmp/tmp.JIkgazmGNg +++ rm /tmp/tmp.FXKTBnSYDA /tmp/tmp.JIkgazmGNg +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6SSXUSs3gm +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ydn8KKsJ7f ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6SSXUSs3gm ++ cat /tmp/tmp.Ydn8KKsJ7f ++ rm /tmp/tmp.6SSXUSs3gm /tmp/tmp.Ydn8KKsJ7f ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + local _orig_key_id=AKIARXP3OARBMEQ6FSEW _orig_secret=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 _orig_default_region= _orig_aws_region= + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z /rs0/myApp.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z + key=/rs0/myApp.test.gz + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z//rs0/myApp.test.gz found in oss-native' Backup operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z//rs0/myApp.test.gz found in oss-native + AWS_ACCESS_KEY_ID=AKIARXP3OARBMEQ6FSEW + AWS_SECRET_ACCESS_KEY=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 + AWS_DEFAULT_REGION= + AWS_REGION= + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + run_recovery_check backup-oss-native some-name-rs0 + local backup=backup-oss-native + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OQ7NgTQefm +++ mktemp ++ local LAST_ERR=/tmp/tmp.oaBoVlThGO ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OQ7NgTQefm ++ cat /tmp/tmp.oaBoVlThGO ++ rm /tmp/tmp.OQ7NgTQefm /tmp/tmp.oaBoVlThGO ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.llB6FismLj ++ mktemp + local LAST_ERR=/tmp/tmp.r98EmF4MQw + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.llB6FismLj Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ef68f395-80f3-454a-8195-419cd27db439") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.r98EmF4MQw + rm /tmp/tmp.llB6FismLj /tmp/tmp.r98EmF4MQw + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:15:32+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TdieviuO1T +++ mktemp ++ local LAST_ERR=/tmp/tmp.DpCVMotPan ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TdieviuO1T ++ cat /tmp/tmp.DpCVMotPan ++ rm /tmp/tmp.TdieviuO1T /tmp/tmp.DpCVMotPan ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.zj3axzAWPj ++ mktemp + local LAST_ERR=/tmp/tmp.hnG5UZfvEP + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zj3axzAWPj + cat /tmp/tmp.hnG5UZfvEP + rm /tmp/tmp.zj3axzAWPj /tmp/tmp.hnG5UZfvEP + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-oss-native + local backup_name=backup-oss-native + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-oss-native' + set +o xtrace [2026-07-17T00:15:35+0000] running restore restore-backup-oss-native + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-oss-native/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-oss-native/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.YxABuipZul ++ mktemp + local LAST_ERR=/tmp/tmp.2QpJ06BWcs + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YxABuipZul perconaservermongodbrestore.psmdb.percona.com/restore-backup-oss-native created + cat /tmp/tmp.2QpJ06BWcs + rm /tmp/tmp.YxABuipZul /tmp/tmp.2QpJ06BWcs + return 0 + wait_restore backup-oss-native some-name + local backup_name=backup-oss-native + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-oss-native object to be created.OK Waiting psmdb-restore/restore-backup-oss-native to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.14BAYAxR5q +++ mktemp ++ local LAST_ERR=/tmp/tmp.tlR2lIYiAD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.14BAYAxR5q ++ cat /tmp/tmp.tlR2lIYiAD ++ rm /tmp/tmp.14BAYAxR5q /tmp/tmp.tlR2lIYiAD ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:16:09+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ir3uiB6uYr +++ mktemp ++ local LAST_ERR=/tmp/tmp.QtXvQX2LmT ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ir3uiB6uYr ++ cat /tmp/tmp.QtXvQX2LmT ++ rm /tmp/tmp.Ir3uiB6uYr /tmp/tmp.QtXvQX2LmT ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.6Dq4VdBaeP ++ mktemp + local LAST_ERR=/tmp/tmp.ybgXzKAp0w + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6Dq4VdBaeP + cat /tmp/tmp.ybgXzKAp0w + rm /tmp/tmp.6Dq4VdBaeP /tmp/tmp.ybgXzKAp0w + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:16:11+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BfTUgmJ5Et +++ mktemp ++ local LAST_ERR=/tmp/tmp.CiQW8rExkl ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BfTUgmJ5Et ++ cat /tmp/tmp.CiQW8rExkl ++ rm /tmp/tmp.BfTUgmJ5Et /tmp/tmp.CiQW8rExkl ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.rcCEyF5jL6 ++ mktemp + local LAST_ERR=/tmp/tmp.diT8FsOHCd + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rcCEyF5jL6 + cat /tmp/tmp.diT8FsOHCd + rm /tmp/tmp.rcCEyF5jL6 /tmp/tmp.diT8FsOHCd + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:16:14+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DrTGVXiDCk +++ mktemp ++ local LAST_ERR=/tmp/tmp.sYQUWTSDSY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DrTGVXiDCk ++ cat /tmp/tmp.sYQUWTSDSY ++ rm /tmp/tmp.DrTGVXiDCk /tmp/tmp.sYQUWTSDSY ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.3NvxHKBsOg ++ mktemp + local LAST_ERR=/tmp/tmp.AGN5biJ4eH + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3NvxHKBsOg + cat /tmp/tmp.AGN5biJ4eH + rm /tmp/tmp.3NvxHKBsOg /tmp/tmp.AGN5biJ4eH + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'check backup and restore -- minio' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- minio ----------------------------------------------------------------------------------- + check_backup_in_storage backup-minio minio rs0 myApp.test.gz + local backup=backup-minio + local storage_type=minio + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.khwiuAhQVl +++ mktemp ++ local LAST_ERR=/tmp/tmp.xRz5PP3QZk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.khwiuAhQVl ++ cat /tmp/tmp.xRz5PP3QZk ++ rm /tmp/tmp.khwiuAhQVl /tmp/tmp.xRz5PP3QZk ++ return 0 + backup_dest=operator-testing/2026-07-17T00:10:49Z + case ${storage_type} in + endpoint=minio-service + aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/2026-07-17T00:10:49Z/rs0/myApp.test.gz + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-07-17T00:10:49Z/rs0/myApp.test.gz + grep myApp.test.gz ++ mktemp + local LAST_OUT=/tmp/tmp.GVTXYsVNuj ++ mktemp + local LAST_ERR=/tmp/tmp.vzn6waaSrz + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/2026-07-17T00:10:49Z/rs0/myApp.test.gz') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-07-17T00:10:49Z/rs0/myApp.test.gz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GVTXYsVNuj + cat /tmp/tmp.vzn6waaSrz All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.GVTXYsVNuj /tmp/tmp.vzn6waaSrz + return 0 2026-07-17 00:10:52 55 myApp.test.gz + run_recovery_check backup-minio some-name-rs0 + local backup=backup-minio + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bLWWj4xiUp +++ mktemp ++ local LAST_ERR=/tmp/tmp.MTjvOhuVhx ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bLWWj4xiUp ++ cat /tmp/tmp.MTjvOhuVhx ++ rm /tmp/tmp.bLWWj4xiUp /tmp/tmp.MTjvOhuVhx ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.H63ZrvrDOw ++ mktemp + local LAST_ERR=/tmp/tmp.Qfl06KZQro + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H63ZrvrDOw Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("92fa9239-1dbb-400e-894a-07c788c03d14") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Qfl06KZQro + rm /tmp/tmp.H63ZrvrDOw /tmp/tmp.Qfl06KZQro + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:16:36+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Iy157lBb4M +++ mktemp ++ local LAST_ERR=/tmp/tmp.O4cKLDZVJS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Iy157lBb4M ++ cat /tmp/tmp.O4cKLDZVJS ++ rm /tmp/tmp.Iy157lBb4M /tmp/tmp.O4cKLDZVJS ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.5APZ7v6J2Z ++ mktemp + local LAST_ERR=/tmp/tmp.0IpsRknQSm + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5APZ7v6J2Z + cat /tmp/tmp.0IpsRknQSm + rm /tmp/tmp.5APZ7v6J2Z /tmp/tmp.0IpsRknQSm + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore backup-minio + local backup_name=backup-minio + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-minio' + set +o xtrace [2026-07-17T00:16:39+0000] running restore restore-backup-minio + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio/' + kubectl_bin apply -f - + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/' ++ mktemp + local LAST_OUT=/tmp/tmp.LGLvxzzIN7 ++ mktemp + local LAST_ERR=/tmp/tmp.PRY9yTv4fj + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LGLvxzzIN7 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created + cat /tmp/tmp.PRY9yTv4fj + rm /tmp/tmp.LGLvxzzIN7 /tmp/tmp.PRY9yTv4fj + return 0 + wait_restore backup-minio some-name + local backup_name=backup-minio + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio object to be created.OK Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RV5Ob4sC3M +++ mktemp ++ local LAST_ERR=/tmp/tmp.tEcRz5CZqS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RV5Ob4sC3M ++ cat /tmp/tmp.tEcRz5CZqS ++ rm /tmp/tmp.RV5Ob4sC3M /tmp/tmp.tEcRz5CZqS ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:17:06+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' +++ mktemp ++ local LAST_OUT=/tmp/tmp.65N96I7vKR +++ mktemp ++ local LAST_ERR=/tmp/tmp.mqGgTuTvJc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.65N96I7vKR ++ cat /tmp/tmp.mqGgTuTvJc ++ rm /tmp/tmp.65N96I7vKR /tmp/tmp.mqGgTuTvJc ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.I4FKjIZ1lX ++ mktemp + local LAST_ERR=/tmp/tmp.pcMWnOZRZy + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I4FKjIZ1lX + cat /tmp/tmp.pcMWnOZRZy + rm /tmp/tmp.I4FKjIZ1lX /tmp/tmp.pcMWnOZRZy + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:17:09+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iMXdp7FRVX +++ mktemp ++ local LAST_ERR=/tmp/tmp.NyShOtZWeP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iMXdp7FRVX ++ cat /tmp/tmp.NyShOtZWeP ++ rm /tmp/tmp.iMXdp7FRVX /tmp/tmp.NyShOtZWeP ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.tZoCsyvlaQ ++ mktemp + local LAST_ERR=/tmp/tmp.HZEuVDblv2 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tZoCsyvlaQ + cat /tmp/tmp.HZEuVDblv2 + rm /tmp/tmp.tZoCsyvlaQ /tmp/tmp.HZEuVDblv2 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:17:11+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W3IX4GNGHk +++ mktemp ++ local LAST_ERR=/tmp/tmp.G1n9oUpzFa ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W3IX4GNGHk ++ cat /tmp/tmp.G1n9oUpzFa ++ rm /tmp/tmp.W3IX4GNGHk /tmp/tmp.G1n9oUpzFa ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.HneMj1DxMd ++ mktemp + local LAST_ERR=/tmp/tmp.mPPVk8ePYh + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HneMj1DxMd + cat /tmp/tmp.mPPVk8ePYh + rm /tmp/tmp.HneMj1DxMd /tmp/tmp.mPPVk8ePYh + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.dropUser("test1user")' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2xxLcgrWNg +++ mktemp ++ local LAST_ERR=/tmp/tmp.XpP8EvIJhk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2xxLcgrWNg ++ cat /tmp/tmp.XpP8EvIJhk ++ rm /tmp/tmp.2xxLcgrWNg /tmp/tmp.XpP8EvIJhk ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kgCbbxZQ1F ++ mktemp + local LAST_ERR=/tmp/tmp.Hso3YDFfy7 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kgCbbxZQ1F Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("849d43b2-c35d-49aa-b984-3586e39e5f8f") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.Hso3YDFfy7 + rm /tmp/tmp.kgCbbxZQ1F /tmp/tmp.Hso3YDFfy7 + return 0 + run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.dropRole("test1role")' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lmmiOKMCbI +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z0XIjZGkiI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lmmiOKMCbI ++ cat /tmp/tmp.Z0XIjZGkiI ++ rm /tmp/tmp.lmmiOKMCbI /tmp/tmp.Z0XIjZGkiI ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.7w8HuRl9uz ++ mktemp + local LAST_ERR=/tmp/tmp.SO9BZzDd5G + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7w8HuRl9uz Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b07b4151-23d0-4413-a0e0-f3eb15fd6b25") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.SO9BZzDd5G + rm /tmp/tmp.7w8HuRl9uz /tmp/tmp.SO9BZzDd5G + return 0 + desc 'selective restore -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore -- minio ----------------------------------------------------------------------------------- + run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local collection=test2 + local with_users_and_roles=false + local username=test1user + local role=test1role + restore_name=restore-backup-minio-selective + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TKpzZvAR2v +++ mktemp ++ local LAST_ERR=/tmp/tmp.r6SFpnunr6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TKpzZvAR2v ++ cat /tmp/tmp.r6SFpnunr6 ++ rm /tmp/tmp.TKpzZvAR2v /tmp/tmp.r6SFpnunr6 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.pVs8oZepni ++ mktemp + local LAST_ERR=/tmp/tmp.2uDqHYpTBw + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pVs8oZepni Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c5d0734b-e12f-4618-a93a-6cc78faea1ae") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.2uDqHYpTBw + rm /tmp/tmp.pVs8oZepni /tmp/tmp.2uDqHYpTBw + return 0 ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp ++ grep -v 'switched to' +++ local LAST_OUT=/tmp/tmp.vclbCydOpD ++ jq 'index("test2") != null' ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BRbxdmSgrt +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vclbCydOpD +++ cat /tmp/tmp.BRbxdmSgrt +++ rm /tmp/tmp.vclbCydOpD /tmp/tmp.BRbxdmSgrt +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VyGxBEqBS7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8mwiMKe5Y2 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VyGxBEqBS7 ++ cat /tmp/tmp.8mwiMKe5Y2 ++ rm /tmp/tmp.VyGxBEqBS7 /tmp/tmp.8mwiMKe5Y2 ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + yq '.metadata.name="restore-backup-minio-selective"' + yq '.spec.backupName="backup-minio"' + yq '.spec.selective.namespaces[0]="myApp.test"' + yq .spec.selective.withUsersAndRoles=false + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.6bhPr9fnHO ++ mktemp + local LAST_ERR=/tmp/tmp.o6BPbxrAqS + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6bhPr9fnHO perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created + cat /tmp/tmp.o6BPbxrAqS + rm /tmp/tmp.6bhPr9fnHO /tmp/tmp.o6BPbxrAqS + return 0 + wait_restore backup-minio-selective some-name + local backup_name=backup-minio-selective + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ofTxdR8oaO +++ mktemp ++ local LAST_ERR=/tmp/tmp.N6bLWlEi96 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ofTxdR8oaO ++ cat /tmp/tmp.N6bLWlEi96 ++ rm /tmp/tmp.ofTxdR8oaO /tmp/tmp.N6bLWlEi96 ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ grep -v 'switched to' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ jq 'index("test2") != null' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ETgcVUQI7V ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W1VhN3fTqn +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ETgcVUQI7V +++ cat /tmp/tmp.W1VhN3fTqn +++ rm /tmp/tmp.ETgcVUQI7V /tmp/tmp.W1VhN3fTqn +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M1dUCNjO6L +++ mktemp ++ local LAST_ERR=/tmp/tmp.t5Bgdbf4tW ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M1dUCNjO6L ++ cat /tmp/tmp.t5Bgdbf4tW ++ rm /tmp/tmp.M1dUCNjO6L /tmp/tmp.t5Bgdbf4tW ++ return 0 + [[ false == true ]] + [[ false == true ]] + [[ false == false ]] ++ user_exists test1user ++ local username=test1user ++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getUsers())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ grep -v 'switched to' ++ jq 'any(.[]; ._id==myApp.test1user)' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4x69DPJU3D ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error +++ local LAST_ERR=/tmp/tmp.8LIe4Z1OHm +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4x69DPJU3D +++ cat /tmp/tmp.8LIe4Z1OHm +++ rm /tmp/tmp.4x69DPJU3D /tmp/tmp.8LIe4Z1OHm +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XPWzyRkWmd +++ mktemp ++ local LAST_ERR=/tmp/tmp.PLID1nFkt6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XPWzyRkWmd ++ cat /tmp/tmp.PLID1nFkt6 ++ rm /tmp/tmp.XPWzyRkWmd /tmp/tmp.PLID1nFkt6 ++ return 0 + [[ '' == true ]] ++ role_exists test1role ++ local role=test1role ++ grep -v 'switched to' ++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 '' '' --quiet ++ jq 'any(.[]; ._id==myApp.test1role)' ++ local 'command=use myApp\n JSON.stringify(db.getRoles())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error +++ local LAST_OUT=/tmp/tmp.extw6KnyiF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JQTPHOOcfe +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.extw6KnyiF +++ cat /tmp/tmp.JQTPHOOcfe +++ rm /tmp/tmp.extw6KnyiF /tmp/tmp.JQTPHOOcfe +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wAFSRpNvuI +++ mktemp ++ local LAST_ERR=/tmp/tmp.tT2bxHERxU ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wAFSRpNvuI ++ cat /tmp/tmp.tT2bxHERxU ++ rm /tmp/tmp.wAFSRpNvuI /tmp/tmp.tT2bxHERxU ++ return 0 + [[ '' == true ]] + kubectl_bin delete psmdb-restore restore-backup-minio-selective ++ mktemp + local LAST_OUT=/tmp/tmp.qH4XmnkSHb ++ mktemp + local LAST_ERR=/tmp/tmp.Hb9ZK7JTMu + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-restore' 'restore-backup-minio-selective') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-restore restore-backup-minio-selective + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qH4XmnkSHb perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-1939 namespace + cat /tmp/tmp.Hb9ZK7JTMu + rm /tmp/tmp.qH4XmnkSHb /tmp/tmp.Hb9ZK7JTMu + return 0 + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tR0MSr0VZK +++ mktemp ++ local LAST_ERR=/tmp/tmp.GpuGnGQ5aN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tR0MSr0VZK ++ cat /tmp/tmp.GpuGnGQ5aN ++ rm /tmp/tmp.tR0MSr0VZK /tmp/tmp.GpuGnGQ5aN ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.v0KytN31ry ++ mktemp + local LAST_ERR=/tmp/tmp.VrXl9JFhe7 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v0KytN31ry Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1f8f8cf9-81c1-43dd-9523-00908fde2eb4") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.VrXl9JFhe7 + rm /tmp/tmp.v0KytN31ry /tmp/tmp.VrXl9JFhe7 + return 0 + desc 'selective restore with users and roles -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore with users and roles -- minio ----------------------------------------------------------------------------------- + run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local collection=test2 + local with_users_and_roles=true + local username=test1user + local role=test1role + restore_name=restore-backup-minio-selective + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CkJfzBpl5H +++ mktemp ++ local LAST_ERR=/tmp/tmp.3OHKKe9b03 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CkJfzBpl5H ++ cat /tmp/tmp.3OHKKe9b03 ++ rm /tmp/tmp.CkJfzBpl5H /tmp/tmp.3OHKKe9b03 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.c1U5QV5QuH ++ mktemp + local LAST_ERR=/tmp/tmp.oOQQBunrP5 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c1U5QV5QuH Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("4c28a964-6228-4004-aac8-d3af94caa28c") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.oOQQBunrP5 + rm /tmp/tmp.c1U5QV5QuH /tmp/tmp.oOQQBunrP5 + return 0 ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ grep -v 'switched to' ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Tr6T0TGR7o ++++ mktemp +++ local LAST_ERR=/tmp/tmp.79VN4gLwUV +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Tr6T0TGR7o +++ cat /tmp/tmp.79VN4gLwUV +++ rm /tmp/tmp.Tr6T0TGR7o /tmp/tmp.79VN4gLwUV +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3mAblSoi91 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3jK4xGWUyU ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3mAblSoi91 ++ cat /tmp/tmp.3jK4xGWUyU ++ rm /tmp/tmp.3mAblSoi91 /tmp/tmp.3jK4xGWUyU ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + yq '.metadata.name="restore-backup-minio-selective"' + yq '.spec.backupName="backup-minio"' + yq .spec.selective.withUsersAndRoles=true + yq '.spec.selective.namespaces[0]="myApp.test"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.jmjR1vZdHL ++ mktemp + local LAST_ERR=/tmp/tmp.rVm2kfarjZ + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jmjR1vZdHL perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created + cat /tmp/tmp.rVm2kfarjZ + rm /tmp/tmp.jmjR1vZdHL /tmp/tmp.rVm2kfarjZ + return 0 + wait_restore backup-minio-selective some-name + local backup_name=backup-minio-selective + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aZGsuGgEB8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.k8bWQaYnSu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aZGsuGgEB8 ++ cat /tmp/tmp.k8bWQaYnSu ++ rm /tmp/tmp.aZGsuGgEB8 /tmp/tmp.k8bWQaYnSu ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ grep -v 'switched to' ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gwvVhKf9l5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0oqJwolPmV +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.gwvVhKf9l5 +++ cat /tmp/tmp.0oqJwolPmV +++ rm /tmp/tmp.gwvVhKf9l5 /tmp/tmp.0oqJwolPmV +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fnO2Uj6E14 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nhXlmCJRkr ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fnO2Uj6E14 ++ cat /tmp/tmp.nhXlmCJRkr ++ rm /tmp/tmp.fnO2Uj6E14 /tmp/tmp.nhXlmCJRkr ++ return 0 + [[ false == true ]] + [[ true == true ]] ++ user_exists test1user ++ local username=test1user ++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getUsers())' ++ grep -v 'switched to' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ jq 'any(.[]; ._id==myApp.test1user)' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pJVD0r10Ay jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HCuNsOG4J3 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.pJVD0r10Ay +++ cat /tmp/tmp.HCuNsOG4J3 +++ rm /tmp/tmp.pJVD0r10Ay /tmp/tmp.HCuNsOG4J3 +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Hu8HUYV7HA +++ mktemp ++ local LAST_ERR=/tmp/tmp.HXw5SIS6tQ ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Hu8HUYV7HA ++ cat /tmp/tmp.HXw5SIS6tQ ++ rm /tmp/tmp.Hu8HUYV7HA /tmp/tmp.HXw5SIS6tQ ++ return 0 + [[ '' == false ]] ++ role_exists test1role ++ local role=test1role ++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 '' '' --quiet ++ jq 'any(.[]; ._id==myApp.test1role)' ++ local 'command=use myApp\n JSON.stringify(db.getRoles())' ++ grep -v 'switched to' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error +++ local LAST_OUT=/tmp/tmp.CJAN6TT0Hy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rJoVptSgyf +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CJAN6TT0Hy +++ cat /tmp/tmp.rJoVptSgyf +++ rm /tmp/tmp.CJAN6TT0Hy /tmp/tmp.rJoVptSgyf +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qEz2Cb5YdK +++ mktemp ++ local LAST_ERR=/tmp/tmp.WAFT1ob1NI ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qEz2Cb5YdK ++ cat /tmp/tmp.WAFT1ob1NI ++ rm /tmp/tmp.qEz2Cb5YdK /tmp/tmp.WAFT1ob1NI ++ return 0 + [[ '' == false ]] + desc 'selective restore with namespace remap -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore with namespace remap -- minio ----------------------------------------------------------------------------------- + run_recovery_check_namespace_remap backup-minio some-name-rs0 myApp test test_remapped + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local src_collection=test + local dst_collection=test_remapped + restore_name=restore-backup-minio-remapped + run_mongo 'use myApp\n db.test_remapped.drop()' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test_remapped.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sp5NKCx7Hm +++ mktemp ++ local LAST_ERR=/tmp/tmp.C8G2VKBUS1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sp5NKCx7Hm ++ cat /tmp/tmp.C8G2VKBUS1 ++ rm /tmp/tmp.sp5NKCx7Hm /tmp/tmp.C8G2VKBUS1 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.c7erT7gU5T ++ mktemp + local LAST_ERR=/tmp/tmp.SbrePknnEW + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c7erT7gU5T Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7be61289-5915-44b1-a422-dc6061ef2a38") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.SbrePknnEW + rm /tmp/tmp.c7erT7gU5T /tmp/tmp.SbrePknnEW + return 0 ++ collection_exists test_remapped ++ local collection=test_remapped ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ grep -v 'switched to' ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ jq 'index("test_remapped") != null' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CKpKRMwRIM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7aS4Ut5SWk +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CKpKRMwRIM +++ cat /tmp/tmp.7aS4Ut5SWk +++ rm /tmp/tmp.CKpKRMwRIM /tmp/tmp.7aS4Ut5SWk +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EVYonSJXNO +++ mktemp ++ local LAST_ERR=/tmp/tmp.RsTGXkFFQ8 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EVYonSJXNO ++ cat /tmp/tmp.RsTGXkFFQ8 ++ rm /tmp/tmp.EVYonSJXNO /tmp/tmp.RsTGXkFFQ8 ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore.yml + yq '.metadata.name="restore-backup-minio-remapped"' + yq '.spec.backupName="backup-minio"' + yq '.spec.selective.nsFrom="myApp.test"' + yq '.spec.selective.nsTo="myApp.test_remapped"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.iGVeJwYeYm ++ mktemp + local LAST_ERR=/tmp/tmp.YrWK2gm8T9 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iGVeJwYeYm perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-remapped created + cat /tmp/tmp.YrWK2gm8T9 + rm /tmp/tmp.iGVeJwYeYm /tmp/tmp.YrWK2gm8T9 + return 0 + wait_restore backup-minio-remapped some-name + local backup_name=backup-minio-remapped + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-remapped object to be created.OK Waiting psmdb-restore/restore-backup-minio-remapped to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a9203hUXj8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OVCLekebgb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.a9203hUXj8 ++ cat /tmp/tmp.OVCLekebgb ++ rm /tmp/tmp.a9203hUXj8 /tmp/tmp.OVCLekebgb ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test_remapped ++ local collection=test_remapped ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ jq 'index("test_remapped") != null' ++ grep -v 'switched to' ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KdCCAwWiLl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.znGUPd4Q07 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KdCCAwWiLl +++ cat /tmp/tmp.znGUPd4Q07 +++ rm /tmp/tmp.KdCCAwWiLl /tmp/tmp.znGUPd4Q07 +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NCVLxpqmWa +++ mktemp ++ local LAST_ERR=/tmp/tmp.D2CPOCBDL6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NCVLxpqmWa ++ cat /tmp/tmp.D2CPOCBDL6 ++ rm /tmp/tmp.NCVLxpqmWa /tmp/tmp.D2CPOCBDL6 ++ return 0 + [[ true == false ]] ++ collection_exists test ++ local collection=test ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-1939 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-1939 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ grep -v 'switched to' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ jq 'index("test") != null' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EGIdv8TeyN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EzDdCCD4AF +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EGIdv8TeyN +++ cat /tmp/tmp.EzDdCCD4AF +++ rm /tmp/tmp.EGIdv8TeyN /tmp/tmp.EzDdCCD4AF +++ return 0 ++ local client_container=psmdb-client-757cd65695-5gfrv ++ kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ph0G4FbJJj +++ mktemp ++ local LAST_ERR=/tmp/tmp.FywyLzvEWw ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ph0G4FbJJj ++ cat /tmp/tmp.FywyLzvEWw ++ rm /tmp/tmp.ph0G4FbJJj /tmp/tmp.FywyLzvEWw ++ return 0 + [[ true == false ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -remapped .svc.cluster.local myApp test_remapped + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-remapped + local suffix=.svc.cluster.local + local database=myApp + local collection=test_remapped + local sort= + local tls=false + local replicaset= + local 'full_command=db.test_remapped.find()' + [[ -n '' ]] + log 'running db.test_remapped.find() in myApp' + set +o xtrace [2026-07-17T00:19:19+0000] running db.test_remapped.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test_remapped.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test_remapped.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v5ln72DWuU +++ mktemp ++ local LAST_ERR=/tmp/tmp.Rz1i0qt0vW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v5ln72DWuU ++ cat /tmp/tmp.Rz1i0qt0vW ++ rm /tmp/tmp.v5ln72DWuU /tmp/tmp.Rz1i0qt0vW ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.feH5aJKE8g ++ mktemp + local LAST_ERR=/tmp/tmp.VDCwqubNq2 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.feH5aJKE8g + cat /tmp/tmp.VDCwqubNq2 + rm /tmp/tmp.feH5aJKE8g /tmp/tmp.VDCwqubNq2 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-remapped.json /tmp/tmp.owL0ZbzDsn/find-remapped + desc 'restore from backup source, with storageName -- minio' + set +o xtrace ----------------------------------------------------------------------------------- restore from backup source, with storageName -- minio ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|oss://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.XJpqzW3YjU +++ mktemp ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.hpPIwkyBsZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XJpqzW3YjU ++ cat /tmp/tmp.hpPIwkyBsZ ++ rm /tmp/tmp.XJpqzW3YjU /tmp/tmp.hpPIwkyBsZ ++ return 0 + backup_dest_minio=operator-testing/2026-07-17T00:10:49Z + run_recovery_check_bkp_source backup-minio operator-testing/2026-07-17T00:10:49Z some-name-rs0 backup-minio-source-0 + local backup=backup-minio + local backup_dest=operator-testing/2026-07-17T00:10:49Z + local cluster=some-name-rs0 + local source=backup-minio-source-0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sWG7AUNKIS +++ mktemp ++ local LAST_ERR=/tmp/tmp.5AJ9HcPFPI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sWG7AUNKIS ++ cat /tmp/tmp.5AJ9HcPFPI ++ rm /tmp/tmp.sWG7AUNKIS /tmp/tmp.5AJ9HcPFPI ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.l5Gwy5gtaU ++ mktemp + local LAST_ERR=/tmp/tmp.s3NRhQxSdg + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l5Gwy5gtaU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("770b8157-f786-4a60-b452-cc033e99981e") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.s3NRhQxSdg + rm /tmp/tmp.l5Gwy5gtaU /tmp/tmp.s3NRhQxSdg + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:19:25+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.esWj75EByQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Dnm2C5Vbti ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.esWj75EByQ ++ cat /tmp/tmp.Dnm2C5Vbti ++ rm /tmp/tmp.esWj75EByQ /tmp/tmp.Dnm2C5Vbti ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ToN58K9QZo ++ mktemp + local LAST_ERR=/tmp/tmp.Nt0mJbqLMd + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ToN58K9QZo + cat /tmp/tmp.Nt0mJbqLMd + rm /tmp/tmp.ToN58K9QZo /tmp/tmp.Nt0mJbqLMd + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore_backupsource backup-minio-source-0 operator-testing/2026-07-17T00:10:49Z + local backupName=backup-minio-source-0 + local backupDest=operator-testing/2026-07-17T00:10:49Z + local storageName= + desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-07-17T00:10:49Z' + set +o xtrace ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-07-17T00:10:49Z ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/' + /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-07-17T00:10:49Z|' + /usr/sbin/sed -e /storageName/d + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.LuFUAj7Pf2 ++ mktemp + local LAST_ERR=/tmp/tmp.1gjNpsYrcQ + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LuFUAj7Pf2 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created + cat /tmp/tmp.1gjNpsYrcQ + rm /tmp/tmp.LuFUAj7Pf2 /tmp/tmp.1gjNpsYrcQ + return 0 + return + wait_restore backup-minio-source-0 some-name + local backup_name=backup-minio-source-0 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nxHY2R5uFz +++ mktemp ++ local LAST_ERR=/tmp/tmp.hSbB8Nyppv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nxHY2R5uFz ++ cat /tmp/tmp.hSbB8Nyppv ++ rm /tmp/tmp.nxHY2R5uFz /tmp/tmp.hSbB8Nyppv ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:19:55+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ObHxNFqCsK +++ mktemp ++ local LAST_ERR=/tmp/tmp.YiLCSmcoRN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ObHxNFqCsK ++ cat /tmp/tmp.YiLCSmcoRN ++ rm /tmp/tmp.ObHxNFqCsK /tmp/tmp.YiLCSmcoRN ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.FlGg24ZidW ++ mktemp + local LAST_ERR=/tmp/tmp.sdrze9zgjs + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FlGg24ZidW + cat /tmp/tmp.sdrze9zgjs + rm /tmp/tmp.FlGg24ZidW /tmp/tmp.sdrze9zgjs + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:19:58+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8tSAB9oKbG +++ mktemp ++ local LAST_ERR=/tmp/tmp.K0PASi6MNM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8tSAB9oKbG ++ cat /tmp/tmp.K0PASi6MNM ++ rm /tmp/tmp.8tSAB9oKbG /tmp/tmp.K0PASi6MNM ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.iEy2l4rSuT ++ mktemp + local LAST_ERR=/tmp/tmp.w1iKxrbb5Q + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iEy2l4rSuT + cat /tmp/tmp.w1iKxrbb5Q + rm /tmp/tmp.iEy2l4rSuT /tmp/tmp.w1iKxrbb5Q + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:20:00+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Laf0WTyeQ1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.YKYumNtYJX ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Laf0WTyeQ1 ++ cat /tmp/tmp.YKYumNtYJX ++ rm /tmp/tmp.Laf0WTyeQ1 /tmp/tmp.YKYumNtYJX ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.c5Z30Qhs6t ++ mktemp + local LAST_ERR=/tmp/tmp.zG1z5vXZtH + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c5Z30Qhs6t + cat /tmp/tmp.zG1z5vXZtH + rm /tmp/tmp.c5Z30Qhs6t /tmp/tmp.zG1z5vXZtH + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'restore from backup source, no storageName -- minio' + set +o xtrace ----------------------------------------------------------------------------------- restore from backup source, no storageName -- minio ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' +++ mktemp ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.Ov2vL8ouEf ++ sed 's|azure://||' +++ mktemp ++ sed 's|oss://||' ++ local LAST_ERR=/tmp/tmp.L6aRFcWQdu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed 's|oci://||' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ov2vL8ouEf ++ cat /tmp/tmp.L6aRFcWQdu ++ rm /tmp/tmp.Ov2vL8ouEf /tmp/tmp.L6aRFcWQdu ++ return 0 + backup_dest_minio=operator-testing/2026-07-17T00:10:49Z + run_recovery_check_bkp_source backup-minio operator-testing/2026-07-17T00:10:49Z some-name-rs0 backup-minio-source-1 + local backup=backup-minio + local backup_dest=operator-testing/2026-07-17T00:10:49Z + local cluster=some-name-rs0 + local source=backup-minio-source-1 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-1939 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c6moGV2IZh +++ mktemp ++ local LAST_ERR=/tmp/tmp.bvGhpbHXXi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c6moGV2IZh ++ cat /tmp/tmp.bvGhpbHXXi ++ rm /tmp/tmp.c6moGV2IZh /tmp/tmp.bvGhpbHXXi ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.KQYhs4qz1e ++ mktemp + local LAST_ERR=/tmp/tmp.WJZ52XRP3R + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KQYhs4qz1e Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("488c6b35-4a3c-4c7d-9af1-5512d5d048d7") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.WJZ52XRP3R + rm /tmp/tmp.KQYhs4qz1e /tmp/tmp.WJZ52XRP3R + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-1939 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:20:06+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-1939 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mDSCFFkIUW +++ mktemp ++ local LAST_ERR=/tmp/tmp.3rEIzyxjvB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mDSCFFkIUW ++ cat /tmp/tmp.3rEIzyxjvB ++ rm /tmp/tmp.mDSCFFkIUW /tmp/tmp.3rEIzyxjvB ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.RuaByBOslK ++ mktemp + local LAST_ERR=/tmp/tmp.jkyKrDFYT7 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RuaByBOslK + cat /tmp/tmp.jkyKrDFYT7 + rm /tmp/tmp.RuaByBOslK /tmp/tmp.jkyKrDFYT7 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.owL0ZbzDsn/find-2nd + run_restore_backupsource backup-minio-source-1 operator-testing/2026-07-17T00:10:49Z + local backupName=backup-minio-source-1 + local backupDest=operator-testing/2026-07-17T00:10:49Z + local storageName= + desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-07-17T00:10:49Z' + set +o xtrace ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-07-17T00:10:49Z ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/' + /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-07-17T00:10:49Z|' + /usr/sbin/sed -e /storageName/d + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.OXqPhjYUGr ++ mktemp + local LAST_ERR=/tmp/tmp.M65v4kd4tj + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OXqPhjYUGr perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created + cat /tmp/tmp.M65v4kd4tj + rm /tmp/tmp.OXqPhjYUGr /tmp/tmp.M65v4kd4tj + return 0 + return + wait_restore backup-minio-source-1 some-name + local backup_name=backup-minio-source-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Sy4EyfoJbQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Qyvk2ucZTg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Sy4EyfoJbQ ++ cat /tmp/tmp.Qyvk2ucZTg ++ rm /tmp/tmp.Sy4EyfoJbQ /tmp/tmp.Qyvk2ucZTg ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:20:34+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xIlfd0Lfpk +++ mktemp ++ local LAST_ERR=/tmp/tmp.YCil1L0Puk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xIlfd0Lfpk ++ cat /tmp/tmp.YCil1L0Puk ++ rm /tmp/tmp.xIlfd0Lfpk /tmp/tmp.YCil1L0Puk ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.gmTQmzKuAV ++ mktemp + local LAST_ERR=/tmp/tmp.cnoSf0JkWj + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gmTQmzKuAV + cat /tmp/tmp.cnoSf0JkWj + rm /tmp/tmp.gmTQmzKuAV /tmp/tmp.cnoSf0JkWj + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:20:37+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.l0ja2O0Kk9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Td5oCFdOF0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.l0ja2O0Kk9 ++ cat /tmp/tmp.Td5oCFdOF0 ++ rm /tmp/tmp.l0ja2O0Kk9 /tmp/tmp.Td5oCFdOF0 ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Pr6x10S519 ++ mktemp + local LAST_ERR=/tmp/tmp.2UqKQYvuem + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Pr6x10S519 + cat /tmp/tmp.2UqKQYvuem + rm /tmp/tmp.Pr6x10S519 /tmp/tmp.2UqKQYvuem + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-07-17T00:20:40+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.COfIyi3amg +++ mktemp ++ local LAST_ERR=/tmp/tmp.J5qgctNHWz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.COfIyi3amg ++ cat /tmp/tmp.J5qgctNHWz ++ rm /tmp/tmp.COfIyi3amg /tmp/tmp.J5qgctNHWz ++ return 0 + local client_container=psmdb-client-757cd65695-5gfrv + kubectl_bin exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Jurh8hyw8U ++ mktemp + local LAST_ERR=/tmp/tmp.sekW6ZYWhz + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-5gfrv' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-5gfrv -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-1939.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Jurh8hyw8U + cat /tmp/tmp.sekW6ZYWhz + rm /tmp/tmp.Jurh8hyw8U /tmp/tmp.sekW6ZYWhz + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/compare/find.json /tmp/tmp.owL0ZbzDsn/find + desc 'delete backup and check if it is removed from bucket -- minio' + set +o xtrace ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.BoIJt9Mby4 ++ mktemp + local LAST_ERR=/tmp/tmp.UC6mjfHSpE + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BoIJt9Mby4 perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-1939 namespace + cat /tmp/tmp.UC6mjfHSpE + rm /tmp/tmp.BoIJt9Mby4 /tmp/tmp.UC6mjfHSpE + return 0 ++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/ ++ local endpoint=http://minio-service:9000 ++ shift ++ grep -c operator-testing/2026-07-17T00:10:49Z ++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ cat +++ mktemp ++ local LAST_OUT=/tmp/tmp.aK0GhUVUql +++ mktemp ++ local LAST_ERR=/tmp/tmp.AXghMxNk8b ++ local exit_status=0 ++ local timeout=4 ++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aK0GhUVUql ++ cat /tmp/tmp.AXghMxNk8b All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-1939 ++ rm /tmp/tmp.aK0GhUVUql /tmp/tmp.AXghMxNk8b ++ return 0 + backup_exists=0 + [[ 0 -eq 1 ]] + '[' -z '' ']' + check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:37Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-07-17T00:10:37Z + key=/rs0/myApp.test.gz + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-07-17T00:10:37Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-07-17T00:10:37Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3' Backup psmdb-demand-backup/2026-07-17T00:10:37Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3 + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:35Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:11:48Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z /rs0/myApp.test.gz ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:10:59Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup/2026-07-17T00:10:59Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob' Backup psmdb-demand-backup/2026-07-17T00:10:59Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob + check_backup_deletion_oss operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1 ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z + key=/rs0/myApp.test.gz ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + region=eu-central-1 + storage_name=oss-native + retry=0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native' Backup psmdb-demand-backup-oss-native/2026-07-17T00:11:12Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native + desc 'checking backup deletion without cr' + set +o xtrace ----------------------------------------------------------------------------------- checking backup deletion without cr ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-07-17T00:21:03+0000] running backup backup-minio + kubectl_bin apply -f - + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.BoxNcBDIXC ++ mktemp + local LAST_ERR=/tmp/tmp.0Z5FhDRaxR + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BoxNcBDIXC perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.0Z5FhDRaxR + rm /tmp/tmp.BoxNcBDIXC /tmp/tmp.0Z5FhDRaxR + return 0 + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-07-17T00:21:05+0000] running backup backup-aws-s3 + yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.MtsR3IWKPm ++ mktemp + local LAST_ERR=/tmp/tmp.p274m9UrcT + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MtsR3IWKPm perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.p274m9UrcT + rm /tmp/tmp.MtsR3IWKPm /tmp/tmp.p274m9UrcT + return 0 + run_backup gcp-cs-s3 + local storage=gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local type=logical + log 'running backup backup-gcp-cs-s3' + set +o xtrace [2026-07-17T00:21:08+0000] running backup backup-gcp-cs-s3 + yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.yEx5kQmxcr ++ mktemp + local LAST_ERR=/tmp/tmp.BJApi7AAkB + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yEx5kQmxcr perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created + cat /tmp/tmp.BJApi7AAkB + rm /tmp/tmp.yEx5kQmxcr /tmp/tmp.BJApi7AAkB + return 0 + run_backup gcp-cs-sa + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=logical + log 'running backup backup-gcp-cs-sa' + set +o xtrace [2026-07-17T00:21:10+0000] running backup backup-gcp-cs-sa + yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.qsCBb3Ewcr ++ mktemp + local LAST_ERR=/tmp/tmp.KA6KsofuYa + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qsCBb3Ewcr perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.KA6KsofuYa + rm /tmp/tmp.qsCBb3Ewcr /tmp/tmp.KA6KsofuYa + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-07-17T00:21:12+0000] running backup backup-azure-blob + yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZT3AdbxVY5 ++ mktemp + local LAST_ERR=/tmp/tmp.Ief9QMi6Nm + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZT3AdbxVY5 perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.Ief9QMi6Nm + rm /tmp/tmp.ZT3AdbxVY5 /tmp/tmp.Ief9QMi6Nm + return 0 + run_backup oss-native + local storage=oss-native + local backup_name=backup-oss-native + local type=logical + log 'running backup backup-oss-native' + set +o xtrace [2026-07-17T00:21:14+0000] running backup backup-oss-native + yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/demand-backup/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.09XXLxdKGV ++ mktemp + local LAST_ERR=/tmp/tmp.RVZ4DAPTWI + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.09XXLxdKGV perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created + cat /tmp/tmp.RVZ4DAPTWI + rm /tmp/tmp.09XXLxdKGV /tmp/tmp.RVZ4DAPTWI + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state..OK + '[' -z '' ']' + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state.................OK + wait_backup backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-s3 to reach ready state............OK + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state.......OK + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state..OK + wait_backup backup-oss-native + local backup_name=backup-oss-native + local target_state=ready + set +o xtrace waiting for backup-oss-native to reach ready state..OK ++ get_backup_dest backup-aws-s3 ++ local backup_name=backup-aws-s3 ++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.ENn86Br69j ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z4qA4a4hpK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ENn86Br69j ++ cat /tmp/tmp.Z4qA4a4hpK ++ rm /tmp/tmp.ENn86Br69j /tmp/tmp.Z4qA4a4hpK ++ return 0 + backup_dest_aws=operator-testing/psmdb-demand-backup/2026-07-17T00:21:39Z ++ get_backup_dest backup-gcp-cs-s3 ++ local backup_name=backup-gcp-cs-s3 ++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' +++ mktemp ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.cJamdRs9qr +++ mktemp ++ local LAST_ERR=/tmp/tmp.VIchBS8DNW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cJamdRs9qr ++ cat /tmp/tmp.VIchBS8DNW ++ rm /tmp/tmp.cJamdRs9qr /tmp/tmp.VIchBS8DNW ++ return 0 + backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z ++ get_backup_dest backup-gcp-cs-sa ++ local backup_name=backup-gcp-cs-sa ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.x9wkblQuES +++ mktemp ++ local LAST_ERR=/tmp/tmp.4o1t6Odvpi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-sa' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.x9wkblQuES ++ cat /tmp/tmp.4o1t6Odvpi ++ rm /tmp/tmp.x9wkblQuES /tmp/tmp.4o1t6Odvpi ++ return 0 + backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ sed -e 's/.json$//' ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' +++ mktemp ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.soI1DDuR3z +++ mktemp ++ local LAST_ERR=/tmp/tmp.PulJeq0uyh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.soI1DDuR3z ++ cat /tmp/tmp.PulJeq0uyh ++ rm /tmp/tmp.soI1DDuR3z /tmp/tmp.PulJeq0uyh ++ return 0 + backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.lPP5GasPDx +++ mktemp ++ local LAST_ERR=/tmp/tmp.NLT0AsXolz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lPP5GasPDx ++ cat /tmp/tmp.NLT0AsXolz ++ rm /tmp/tmp.lPP5GasPDx /tmp/tmp.NLT0AsXolz ++ return 0 + backup_dest_oss_native=operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ujEXsXPRpK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3FglmdjlvS +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ujEXsXPRpK +++ cat /tmp/tmp.3FglmdjlvS +++ rm /tmp/tmp.ujEXsXPRpK /tmp/tmp.3FglmdjlvS +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XXKz5gIXtX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3iYCW3082f +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.XXKz5gIXtX +++ cat /tmp/tmp.3iYCW3082f +++ rm /tmp/tmp.XXKz5gIXtX /tmp/tmp.3iYCW3082f +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DRhzeili4N +++ mktemp ++ local LAST_ERR=/tmp/tmp.alOjyF8Xio ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DRhzeili4N ++ cat /tmp/tmp.alOjyF8Xio ++ rm /tmp/tmp.DRhzeili4N /tmp/tmp.alOjyF8Xio ++ return 0 + backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eYPTKfv0pH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qUfgoh9cOn +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.eYPTKfv0pH +++ cat /tmp/tmp.qUfgoh9cOn +++ rm /tmp/tmp.eYPTKfv0pH /tmp/tmp.qUfgoh9cOn +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.byJB2MH9RD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sqSH5uUdie +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.byJB2MH9RD +++ cat /tmp/tmp.sqSH5uUdie +++ rm /tmp/tmp.byJB2MH9RD /tmp/tmp.sqSH5uUdie +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iFBdXhgV4n +++ mktemp ++ local LAST_ERR=/tmp/tmp.O0o6H3H3Us ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iFBdXhgV4n ++ cat /tmp/tmp.O0o6H3H3Us ++ rm /tmp/tmp.iFBdXhgV4n /tmp/tmp.O0o6H3H3Us ++ return 0 + backup_region_oss_native=eu-central-1 + kubectl_bin delete psmdb --all ++ mktemp + local LAST_OUT=/tmp/tmp.cPZJ25EI0L ++ mktemp + local LAST_ERR=/tmp/tmp.VNqMox8RDL + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cPZJ25EI0L perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-1939 namespace + cat /tmp/tmp.VNqMox8RDL + rm /tmp/tmp.cPZJ25EI0L /tmp/tmp.VNqMox8RDL + return 0 + sleep 60 + desc 'delete backup and check if it is removed from bucket -- minio' + set +o xtrace ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.wqwcnPivMv ++ mktemp + local LAST_ERR=/tmp/tmp.xVgvnuJCSE + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wqwcnPivMv perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-1939 namespace perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-1939 namespace + cat /tmp/tmp.xVgvnuJCSE + rm /tmp/tmp.wqwcnPivMv /tmp/tmp.xVgvnuJCSE + return 0 ++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/ ++ cat ++ grep -c operator-testing/2026-07-17T00:10:49Z ++ local endpoint=http://minio-service:9000 ++ shift ++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ +++ mktemp ++ local LAST_OUT=/tmp/tmp.0JSd1FZRe2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JV6wh4RxiR ++ local exit_status=0 ++ local timeout=4 ++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0JSd1FZRe2 ++ cat /tmp/tmp.JV6wh4RxiR All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-1939 ++ rm /tmp/tmp.0JSd1FZRe2 /tmp/tmp.JV6wh4RxiR ++ return 0 + backup_exists=0 + [[ 0 -eq 1 ]] + '[' -z '' ']' + check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-07-17T00:21:39Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:21:39Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:21:39Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-07-17T00:21:39Z + key=/rs0/myApp.test.gz + storage_name=aws-s3 + retry=0 + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-07-17T00:21:39Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-07-17T00:21:39Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3' Backup psmdb-demand-backup/2026-07-17T00:21:39Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3 + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:01Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-07-17T00:22:13Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z /rs0/myApp.test.gz ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z + url_path=operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z ++ cut -d/ -f1 + container=operator-testing ++ cut -d/ -f2- ++ echo operator-testing/psmdb-demand-backup/2026-07-17T00:21:50Z + blob_prefix=psmdb-demand-backup/2026-07-17T00:21:50Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-07-17T00:21:50Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-07-17T00:21:50Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-07-17T00:21:50Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob' Backup psmdb-demand-backup/2026-07-17T00:21:50Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob + check_backup_deletion_oss operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1 ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z + key=/rs0/myApp.test.gz ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + region=eu-central-1 + storage_name=oss-native + retry=0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native' Backup psmdb-demand-backup-oss-native/2026-07-17T00:21:17Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native + destroy demand-backup-1939 + local namespace=demand-backup-1939 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.OBRU7qtiOj +++ mktemp ++ local LAST_ERR=/tmp/tmp.wE50FBDKta ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OBRU7qtiOj ++ cat /tmp/tmp.wE50FBDKta No resources found in demand-backup-1939 namespace. ++ rm /tmp/tmp.OBRU7qtiOj /tmp/tmp.wE50FBDKta ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.1BfpHvjh0l ++ mktemp + local LAST_ERR=/tmp/tmp.B8nyVGshTd + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1BfpHvjh0l customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.B8nyVGshTd + rm /tmp/tmp.1BfpHvjh0l /tmp/tmp.B8nyVGshTd + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.lRXBG0vlkY ++ mktemp + local LAST_ERR=/tmp/tmp.XymdLRWL6W + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lRXBG0vlkY + cat /tmp/tmp.XymdLRWL6W + rm /tmp/tmp.lRXBG0vlkY /tmp/tmp.XymdLRWL6W + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Efd1PNBHoz ++ mktemp + local LAST_ERR=/tmp/tmp.rlC5byytsW + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Efd1PNBHoz + cat /tmp/tmp.rlC5byytsW + rm /tmp/tmp.Efd1PNBHoz /tmp/tmp.rlC5byytsW + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.v1ZHcGGeRl ++ mktemp + local LAST_ERR=/tmp/tmp.dNwwDVS0Z4 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v1ZHcGGeRl + cat /tmp/tmp.dNwwDVS0Z4 + rm /tmp/tmp.v1ZHcGGeRl /tmp/tmp.dNwwDVS0Z4 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.lAECYBWpfD ++ mktemp + local LAST_ERR=/tmp/tmp.Nl66oCCa0b + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lAECYBWpfD + cat /tmp/tmp.Nl66oCCa0b + rm /tmp/tmp.lAECYBWpfD /tmp/tmp.Nl66oCCa0b + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.yxUv5hY3Pj ++ mktemp + local LAST_ERR=/tmp/tmp.rJKH6DFrYl + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yxUv5hY3Pj clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.rJKH6DFrYl + rm /tmp/tmp.yxUv5hY3Pj /tmp/tmp.rJKH6DFrYl + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2ciJuoGkj6 ++ mktemp + local LAST_ERR=/tmp/tmp.o2OCt8YiHN + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.2ciJuoGkj6 + cat /tmp/tmp.o2OCt8YiHN Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.2ciJuoGkj6 + cat /tmp/tmp.o2OCt8YiHN Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.2ciJuoGkj6 + cat /tmp/tmp.o2OCt8YiHN Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.2ciJuoGkj6 + cat /tmp/tmp.o2OCt8YiHN Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.2ciJuoGkj6 /tmp/tmp.o2OCt8YiHN + return 1 + true + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-1939 + rm -rf /tmp/tmp.owL0ZbzDsn + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.IdoK2yD8Im ++ mktemp PASSED+ local LAST_OUT=/tmp/tmp.1LnHFx2WfW ++ mktemp + local LAST_ERR=/tmp/tmp.clc6thNcTl + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + local LAST_ERR=/tmp/tmp.CHVpfR8wB8 + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'demand-backup-1939') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace demand-backup-1939 - generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/demand-backup.xml - - Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/demand-backup.html - ======================== 1 passed in 1401.71s (0:23:21) ======================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1LnHFx2WfW namespace "psmdb-operator" force deleted + cat /tmp/tmp.clc6thNcTl Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.1LnHFx2WfW /tmp/tmp.clc6thNcTl + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IdoK2yD8Im namespace "demand-backup-1939" force deleted + cat /tmp/tmp.CHVpfR8wB8 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.IdoK2yD8Im /tmp/tmp.CHVpfR8wB8 + return 0