which: no gsed in (/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) mkdir -p e2e-tests/reports e2e-tests/logs ========================================================================================= test session starts ========================================================================================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-6.18.3-200.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'dependency': '0.6.1', 'json-report': '1.5.0', 'metadata': '3.1.1', 'html': '4.2.0'}, 'CI': 'true', 'BUILD_NUMBER': '3', 'BUILD_ID': '3', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2491/3/', 'NODE_NAME': 'hcloud-tfqjtxqnhrf6stbu', 'JOB_NAME': 'cloud-psmdb-operator/PR-2491', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2491-3', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491', 'GIT_COMMIT': '6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2491'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491 configfile: pyproject.toml plugins: dependency-0.6.1, json-report-1.5.0, metadata-3.1.1, html-4.2.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[demand-backup] [19:43:05.229475] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [19:43:06.515288] INFO Environment variables: KUBE_VERSION=1.33 PLATFORM=gke ARCH= EKS= GKE=1 AKS= OPENSHIFT= KIND= MINIKUBE= RANCHER= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a GIT_BRANCH=PR-2491 OPERATOR_VERSION=1.24.0 IMAGE=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 IMAGE_MONGOD=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_SEARCH=docker.io/perconalab/percona-server-mongodb-operator:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mongodb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1-1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-dev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersync-mongodb:0.9.0 IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 CERT_MANAGER_VER=1.21.0 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + create_infra demand-backup-26283 + local ns=demand-backup-26283 + echo demand-backup-26283 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.WxpgSWcZ70 ++ mktemp + local LAST_ERR=/tmp/tmp.RuXslEP1Kd + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WxpgSWcZ70 + cat /tmp/tmp.RuXslEP1Kd + rm /tmp/tmp.WxpgSWcZ70 /tmp/tmp.RuXslEP1Kd + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.dCHVzYYsGj ++ mktemp + local LAST_ERR=/tmp/tmp.IEitf5NZAN + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dCHVzYYsGj + cat /tmp/tmp.IEitf5NZAN + rm /tmp/tmp.dCHVzYYsGj /tmp/tmp.IEitf5NZAN + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.xhDPRS0jYM ++ mktemp + local LAST_ERR=/tmp/tmp.8DVxZ084JH + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xhDPRS0jYM + cat /tmp/tmp.8DVxZ084JH + rm /tmp/tmp.xhDPRS0jYM /tmp/tmp.8DVxZ084JH + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.hVJhmI06D1 ++ mktemp + local LAST_ERR=/tmp/tmp.jDdAnrrYPg + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hVJhmI06D1 + cat /tmp/tmp.jDdAnrrYPg + rm /tmp/tmp.hVJhmI06D1 /tmp/tmp.jDdAnrrYPg + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ZfGztxS9ZX ++ mktemp + local LAST_ERR=/tmp/tmp.Jqnq3UI0DX + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZfGztxS9ZX + cat /tmp/tmp.Jqnq3UI0DX + rm /tmp/tmp.ZfGztxS9ZX /tmp/tmp.Jqnq3UI0DX + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1V5GnFpiwA ++ mktemp + local LAST_ERR=/tmp/tmp.ETolf2EBvf + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1V5GnFpiwA + cat /tmp/tmp.ETolf2EBvf + rm /tmp/tmp.1V5GnFpiwA /tmp/tmp.ETolf2EBvf + return 0 + check_crd_for_deletion PR-2491-6cfab0ed6 + local git_tag=PR-2491-6cfab0ed6 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2491-6cfab0ed6/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ml2kAnmOEB +++ mktemp ++ local LAST_ERR=/tmp/tmp.0Rh3KDO9R2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ml2kAnmOEB ++ cat /tmp/tmp.0Rh3KDO9R2 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ml2kAnmOEB ++ cat /tmp/tmp.0Rh3KDO9R2 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ml2kAnmOEB ++ cat /tmp/tmp.0Rh3KDO9R2 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Ml2kAnmOEB ++ cat /tmp/tmp.0Rh3KDO9R2 Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Ml2kAnmOEB /tmp/tmp.0Rh3KDO9R2 ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + xargs kubectl delete ns ++ mktemp + [[ -n '' ]] + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.w2OMLcXTPb ++ mktemp + local LAST_OUT=/tmp/tmp.17u2SPEXWY ++ mktemp + local LAST_ERR=/tmp/tmp.lmqaQR7YLS + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.6gc82f6pdu + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') ++ seq 0 2 + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.w2OMLcXTPb + cat /tmp/tmp.lmqaQR7YLS + rm /tmp/tmp.w2OMLcXTPb /tmp/tmp.lmqaQR7YLS + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.17u2SPEXWY + cat /tmp/tmp.6gc82f6pdu + rm /tmp/tmp.17u2SPEXWY /tmp/tmp.6gc82f6pdu + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ou75JaiWIA ++ mktemp + local LAST_ERR=/tmp/tmp.G40AFoH1xR + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ou75JaiWIA + cat /tmp/tmp.G40AFoH1xR + rm /tmp/tmp.ou75JaiWIA /tmp/tmp.G40AFoH1xR + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.OLW9N6VDpe ++ mktemp + local LAST_ERR=/tmp/tmp.q2CfmCLC6b + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OLW9N6VDpe namespace/psmdb-operator created + cat /tmp/tmp.q2CfmCLC6b + rm /tmp/tmp.OLW9N6VDpe /tmp/tmp.q2CfmCLC6b + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.oHj1zIFudc +++ mktemp ++ local LAST_ERR=/tmp/tmp.yuJfJlhT7E ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oHj1zIFudc ++ cat /tmp/tmp.yuJfJlhT7E ++ rm /tmp/tmp.oHj1zIFudc /tmp/tmp.yuJfJlhT7E ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.yLwRxVZsuV ++ mktemp + local LAST_ERR=/tmp/tmp.S3UAf3m5kW + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yLwRxVZsuV Context "gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4" modified. + cat /tmp/tmp.S3UAf3m5kW + rm /tmp/tmp.yLwRxVZsuV /tmp/tmp.S3UAf3m5kW + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.lj0MIc91JB ++ mktemp + local LAST_ERR=/tmp/tmp.c3Ewv5evE4 + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lj0MIc91JB customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.c3Ewv5evE4 + rm /tmp/tmp.lj0MIc91JB /tmp/tmp.c3Ewv5evE4 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.RIapyVpBxO ++ mktemp + local LAST_ERR=/tmp/tmp.vKzFmLQDzQ + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RIapyVpBxO clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.vKzFmLQDzQ + rm /tmp/tmp.RIapyVpBxO /tmp/tmp.vKzFmLQDzQ + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-operator.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.nG4FNdXZY0 ++ mktemp + local LAST_ERR=/tmp/tmp.rtJqrqwId1 + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nG4FNdXZY0 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.rtJqrqwId1 + rm /tmp/tmp.nG4FNdXZY0 /tmp/tmp.rtJqrqwId1 + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.xZzr2iGMHq +++ mktemp ++ local LAST_ERR=/tmp/tmp.tvDG3nms54 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xZzr2iGMHq ++ cat /tmp/tmp.tvDG3nms54 ++ rm /tmp/tmp.xZzr2iGMHq /tmp/tmp.tvDG3nms54 ++ return 0 + wait_operator_pod percona-server-mongodb-operator-78964f6749-qvxgx + local pod=percona-server-mongodb-operator-78964f6749-qvxgx + set +o xtrace waiting for pod/percona-server-mongodb-operator-78964f6749-qvxgx to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.kjmoJ92aKT +++ mktemp ++ local LAST_ERR=/tmp/tmp.iFtAPjUWxS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kjmoJ92aKT ++ cat /tmp/tmp.iFtAPjUWxS ++ rm /tmp/tmp.kjmoJ92aKT /tmp/tmp.iFtAPjUWxS ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-78964f6749-qvxgx ++ mktemp + local LAST_OUT=/tmp/tmp.yTr3JjBUAT ++ mktemp + local LAST_ERR=/tmp/tmp.To7EEYUzFe + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-78964f6749-qvxgx') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-78964f6749-qvxgx + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yTr3JjBUAT + cat /tmp/tmp.To7EEYUzFe + rm /tmp/tmp.yTr3JjBUAT /tmp/tmp.To7EEYUzFe + return 0 2026-08-04T19:44:10.356Z INFO setup Manager starting up {"gitCommit": "6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a", "gitBranch": "PR-2491-6cfab0ed6", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-26283 + local namespace=demand-backup-26283 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] + xargs kubectl delete ns + desc 'cleaned up old namespaces demand-backup-26283' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-26283 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-26283 --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.914EdbwyDu ++ mktemp + local LAST_OUT=/tmp/tmp.S8JEkx8EiU + local LAST_ERR=/tmp/tmp.tyKQkkMLve + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.dAnTpAsdan + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'demand-backup-26283' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-26283 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.914EdbwyDu + cat /tmp/tmp.tyKQkkMLve + rm /tmp/tmp.914EdbwyDu /tmp/tmp.tyKQkkMLve + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S8JEkx8EiU + cat /tmp/tmp.dAnTpAsdan + rm /tmp/tmp.S8JEkx8EiU /tmp/tmp.dAnTpAsdan + return 0 + kubectl_bin wait --for=delete namespace demand-backup-26283 ++ mktemp + local LAST_OUT=/tmp/tmp.THlC9XpbPm ++ mktemp + local LAST_ERR=/tmp/tmp.a4emwjWum3 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'demand-backup-26283') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-26283 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.THlC9XpbPm + cat /tmp/tmp.a4emwjWum3 + rm /tmp/tmp.THlC9XpbPm /tmp/tmp.a4emwjWum3 + return 0 + desc 'create namespace demand-backup-26283' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-26283 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-26283 ++ mktemp + local LAST_OUT=/tmp/tmp.xvqHpW69HJ ++ mktemp + local LAST_ERR=/tmp/tmp.kJRIjWPry2 + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'demand-backup-26283') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-26283 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xvqHpW69HJ namespace/demand-backup-26283 created + cat /tmp/tmp.kJRIjWPry2 + rm /tmp/tmp.xvqHpW69HJ /tmp/tmp.kJRIjWPry2 + return 0 + set_kube_ctx demand-backup-26283 + local namespace=demand-backup-26283 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.gxXJRrnqpV +++ mktemp ++ local LAST_ERR=/tmp/tmp.jvROSjtYXY ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gxXJRrnqpV ++ cat /tmp/tmp.jvROSjtYXY ++ rm /tmp/tmp.gxXJRrnqpV /tmp/tmp.jvROSjtYXY ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4 --namespace=demand-backup-26283 ++ mktemp + local LAST_OUT=/tmp/tmp.fDFszJdMAo ++ mktemp + local LAST_ERR=/tmp/tmp.VFDWV8RLVc + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4' '--namespace=demand-backup-26283') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4 --namespace=demand-backup-26283 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fDFszJdMAo Context "gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster4" modified. + cat /tmp/tmp.VFDWV8RLVc + rm /tmp/tmp.fDFszJdMAo /tmp/tmp.VFDWV8RLVc + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + arch_scheduling_enabled + [[ -n '' ]] + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Tue Aug 4 19:44:45 2026 NAMESPACE: demand-backup-26283 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-26283.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-26283 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-26283 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-26283 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-26283 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.g3Tt2VZ5Qe +++ mktemp ++ local LAST_ERR=/tmp/tmp.lQcyfSU4wu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=release=minio-service' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.g3Tt2VZ5Qe ++ cat /tmp/tmp.lQcyfSU4wu ++ rm /tmp/tmp.g3Tt2VZ5Qe /tmp/tmp.lQcyfSU4wu ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-l5x9b + wait_pod minio-service-6d5f646cdc-l5x9b + local pod=minio-service-6d5f646cdc-l5x9b + set +o xtrace waiting for pod/minio-service-6d5f646cdc-l5x9b to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-26283.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + aws_cli_minio http://minio-service:9000 s3 mb s3://operator-testing + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.CtMLqNGMrl ++ mktemp + local LAST_ERR=/tmp/tmp.QPBoeYTqZB + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'mb' 's3://operator-testing') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CtMLqNGMrl pod "aws-cli" deleted from demand-backup-26283 namespace + cat /tmp/tmp.QPBoeYTqZB All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.CtMLqNGMrl /tmp/tmp.QPBoeYTqZB + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + cluster=some-name-rs0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.9Bq00Sp9hL ++ mktemp + local LAST_ERR=/tmp/tmp.6bcqBTgfPo + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9Bq00Sp9hL secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.6bcqBTgfPo + rm /tmp/tmp.9Bq00Sp9hL /tmp/tmp.6bcqBTgfPo + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.XTvpkwqhyU ++ mktemp + local LAST_ERR=/tmp/tmp.ROm3DyFNwi + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XTvpkwqhyU secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/oci-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.ROm3DyFNwi + rm /tmp/tmp.XTvpkwqhyU /tmp/tmp.ROm3DyFNwi + return 0 + desc 'create first PSMDB cluster some-name-rs0' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name-rs0 ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + local LAST_OUT=/tmp/tmp.uc6c4PVT3o + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-26283/g + add_arch_to_psmdb - + yq eval '(.spec | select(has("search"))).search.image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongot"' + arch_scheduling_enabled + [[ -n '' ]] + cat - ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.FS68N6Vizn + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uc6c4PVT3o perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.FS68N6Vizn + rm /tmp/tmp.uc6c4PVT3o /tmp/tmp.FS68N6Vizn + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready..................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready................OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C1uwemvfT1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RULpb9krl7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C1uwemvfT1 ++ cat /tmp/tmp.RULpb9krl7 ++ rm /tmp/tmp.C1uwemvfT1 /tmp/tmp.RULpb9krl7 ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...................OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HiJBvfAKRI +++ mktemp ++ local LAST_ERR=/tmp/tmp.GZ5CW9jTeW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HiJBvfAKRI ++ cat /tmp/tmp.GZ5CW9jTeW ++ rm /tmp/tmp.HiJBvfAKRI /tmp/tmp.GZ5CW9jTeW ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v4E47mLAy7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RKYK6udhPn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v4E47mLAy7 ++ cat /tmp/tmp.RKYK6udhPn ++ rm /tmp/tmp.v4E47mLAy7 /tmp/tmp.RKYK6udhPn ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready... + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.UFgFtUGm4s/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-26283", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.O6NYpy8fPr ++ mktemp + local LAST_ERR=/tmp/tmp.4br4Z1DUQX + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.O6NYpy8fPr + cat /tmp/tmp.4br4Z1DUQX + rm /tmp/tmp.O6NYpy8fPr /tmp/tmp.4br4Z1DUQX + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.UFgFtUGm4s/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.UFgFtUGm4s/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.UFgFtUGm4s/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.UFgFtUGm4s/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-08-04T19:48:07+0000] compare_kubectl: statefulset/some-name-rs0 OK + check_custom_backup_probes some-name-rs0 + local cluster=some-name-rs0 + desc 'verify custom backup-agent liveness/readiness probes set via CR are applied' + set +o xtrace ----------------------------------------------------------------------------------- verify custom backup-agent liveness/readiness probes set via CR are applied ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.r81fqHGTM1 ++ mktemp + local LAST_ERR=/tmp/tmp.dw7e4VV2ms + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[ {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}}, {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json $'-p=[\n\t {"op":"add","path":"/spec/backup/livenessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":30,"periodSeconds":15}},\n\t {"op":"add","path":"/spec/backup/readinessProbe","value":{"exec":{"command":["/bin/true"]},"initialDelaySeconds":5,"periodSeconds":10}}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.r81fqHGTM1 perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.dw7e4VV2ms + rm /tmp/tmp.r81fqHGTM1 /tmp/tmp.dw7e4VV2ms + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2vmwZSOhsQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.p0CxofmTBJ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2vmwZSOhsQ ++ cat /tmp/tmp.p0CxofmTBJ ++ rm /tmp/tmp.2vmwZSOhsQ /tmp/tmp.p0CxofmTBJ ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2sfEHZak2T +++ mktemp ++ local LAST_ERR=/tmp/tmp.W6URiHg00w ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2sfEHZak2T ++ cat /tmp/tmp.W6URiHg00w ++ rm /tmp/tmp.2sfEHZak2T /tmp/tmp.W6URiHg00w ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iYiz4OnRrp +++ mktemp ++ local LAST_ERR=/tmp/tmp.YxFGHO92VM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iYiz4OnRrp ++ cat /tmp/tmp.YxFGHO92VM ++ rm /tmp/tmp.iYiz4OnRrp /tmp/tmp.YxFGHO92VM ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.................................................. + local liveness readiness ++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.67M3zKucYV +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgzAUGucRh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].livenessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.67M3zKucYV ++ cat /tmp/tmp.bgzAUGucRh ++ rm /tmp/tmp.67M3zKucYV /tmp/tmp.bgzAUGucRh ++ return 0 + liveness=/bin/true ++ kubectl_bin get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gP3h1LDY1B +++ mktemp ++ local LAST_ERR=/tmp/tmp.yf2PSHPqmq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset/some-name-rs0' '-o' 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset/some-name-rs0 -o 'jsonpath={.spec.template.spec.containers[?(@.name=="backup-agent")].readinessProbe.exec.command[0]}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gP3h1LDY1B ++ cat /tmp/tmp.yf2PSHPqmq ++ rm /tmp/tmp.gP3h1LDY1B /tmp/tmp.yf2PSHPqmq ++ return 0 + readiness=/bin/true + [[ /bin/true != /bin/true ]] + [[ /bin/true != /bin/true ]] + desc 'revert custom backup-agent probes to defaults' + set +o xtrace ----------------------------------------------------------------------------------- revert custom backup-agent probes to defaults ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=json $'-p=[\n\t {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]' ++ mktemp + local LAST_OUT=/tmp/tmp.CyBU0kjo24 ++ mktemp + local LAST_ERR=/tmp/tmp.e9ejK6yNP7 + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=json' '-p=[ {"op":"remove","path":"/spec/backup/livenessProbe"}, {"op":"remove","path":"/spec/backup/readinessProbe"} ]') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=json $'-p=[\n\t {"op":"remove","path":"/spec/backup/livenessProbe"},\n\t {"op":"remove","path":"/spec/backup/readinessProbe"}\n\t]' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CyBU0kjo24 perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.e9ejK6yNP7 + rm /tmp/tmp.CyBU0kjo24 /tmp/tmp.e9ejK6yNP7 + return 0 + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LhCxcjUoSf +++ mktemp ++ local LAST_ERR=/tmp/tmp.sxVVaNETnN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LhCxcjUoSf ++ cat /tmp/tmp.sxVVaNETnN ++ rm /tmp/tmp.LhCxcjUoSf /tmp/tmp.sxVVaNETnN ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6A70I3DuGj +++ mktemp ++ local LAST_ERR=/tmp/tmp.WD3NTzw4Ue ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6A70I3DuGj ++ cat /tmp/tmp.WD3NTzw4Ue ++ rm /tmp/tmp.6A70I3DuGj /tmp/tmp.WD3NTzw4Ue ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0ENbGp3tGW +++ mktemp ++ local LAST_ERR=/tmp/tmp.W8j92mSakX ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0ENbGp3tGW ++ cat /tmp/tmp.W8j92mSakX ++ rm /tmp/tmp.0ENbGp3tGW /tmp/tmp.W8j92mSakX ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.................................................. + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7gzm6wAEYE +++ mktemp ++ local LAST_ERR=/tmp/tmp.u5p76GkMiI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7gzm6wAEYE ++ cat /tmp/tmp.u5p76GkMiI ++ rm /tmp/tmp.7gzm6wAEYE /tmp/tmp.u5p76GkMiI ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.NuEEGTrfZQ ++ mktemp + local LAST_ERR=/tmp/tmp.BxF05N5vKU + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NuEEGTrfZQ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("fc2675bb-6578-4dc4-a302-cf78275c7e02") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.BxF05N5vKU + rm /tmp/tmp.NuEEGTrfZQ /tmp/tmp.BxF05N5vKU + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TAAz70O1tF +++ mktemp ++ local LAST_ERR=/tmp/tmp.RFlkFX0S6b ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TAAz70O1tF ++ cat /tmp/tmp.RFlkFX0S6b ++ rm /tmp/tmp.TAAz70O1tF /tmp/tmp.RFlkFX0S6b ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.JgnVS39Rc6 ++ mktemp + local LAST_ERR=/tmp/tmp.tDEZDJAzQQ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JgnVS39Rc6 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("19cd12e4-be96-4d18-b4d0-42d2db6368a8") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.tDEZDJAzQQ + rm /tmp/tmp.JgnVS39Rc6 /tmp/tmp.tDEZDJAzQQ + return 0 + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xJY1lta8lK +++ mktemp ++ local LAST_ERR=/tmp/tmp.sXqTrruAHk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xJY1lta8lK ++ cat /tmp/tmp.sXqTrruAHk ++ rm /tmp/tmp.xJY1lta8lK /tmp/tmp.sXqTrruAHk ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.d1Kpwb2WEQ ++ mktemp + local LAST_ERR=/tmp/tmp.egV1wafbge + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.d1Kpwb2WEQ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("bee1e156-c42e-4766-83c7-9cf5ee7a6160") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.egV1wafbge + rm /tmp/tmp.d1Kpwb2WEQ /tmp/tmp.egV1wafbge + return 0 + run_mongo 'use myApp\n db.test3.insert({ x: 100502 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test3.insert({ x: 100502 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gIl895kUqD +++ mktemp ++ local LAST_ERR=/tmp/tmp.Pkzmh7Pkmc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gIl895kUqD ++ cat /tmp/tmp.Pkzmh7Pkmc ++ rm /tmp/tmp.gIl895kUqD /tmp/tmp.Pkzmh7Pkmc ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.L0uhZaM7ii ++ mktemp + local LAST_ERR=/tmp/tmp.Csu7tb3e1g + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test3.insert({ x: 100502 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L0uhZaM7ii Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("caf158e0-4c5e-4453-8ed8-7bb13a334ae2") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Csu7tb3e1g + rm /tmp/tmp.L0uhZaM7ii /tmp/tmp.Csu7tb3e1g + return 0 + custom_user_name=test1user + custom_role_name=test1role + run_mongo 'use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3Tnn0GrvZU +++ mktemp ++ local LAST_ERR=/tmp/tmp.UNbeHZC14X ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3Tnn0GrvZU ++ cat /tmp/tmp.UNbeHZC14X ++ rm /tmp/tmp.3Tnn0GrvZU /tmp/tmp.UNbeHZC14X ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.zoDdyjeI3n ++ mktemp + local LAST_ERR=/tmp/tmp.rSbRXCYYBK + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createUser({user: "test1user",pwd:"test1pass",roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zoDdyjeI3n Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("790be58e-3d31-4c26-bd55-d27e3939652c") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp Successfully added user: { "user" : "test1user", "roles" : [ ] } bye + cat /tmp/tmp.rSbRXCYYBK + rm /tmp/tmp.zoDdyjeI3n /tmp/tmp.rSbRXCYYBK + return 0 + run_mongo 'use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S1UorxdfcW +++ mktemp ++ local LAST_ERR=/tmp/tmp.bP4PeJh6Mg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.S1UorxdfcW ++ cat /tmp/tmp.bP4PeJh6Mg ++ rm /tmp/tmp.S1UorxdfcW /tmp/tmp.bP4PeJh6Mg ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.sywBcV4H0O ++ mktemp + local LAST_ERR=/tmp/tmp.aE3iW6is9A + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.createRole({"role":"test1role", privileges:[],roles:[]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sywBcV4H0O Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("9f02c083-c4b0-4ce3-8f6a-a7dfd60f10f8") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp { "role" : "test1role", "privileges" : [ ], "roles" : [ ] } bye + cat /tmp/tmp.aE3iW6is9A + rm /tmp/tmp.sywBcV4H0O /tmp/tmp.aE3iW6is9A + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:52:37+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qx0WbMd32a +++ mktemp ++ local LAST_ERR=/tmp/tmp.1D8EyQRKj1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qx0WbMd32a ++ cat /tmp/tmp.1D8EyQRKj1 ++ rm /tmp/tmp.qx0WbMd32a /tmp/tmp.1D8EyQRKj1 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.26TkyTt7NZ ++ mktemp + local LAST_ERR=/tmp/tmp.gIPcXfzwGy + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.26TkyTt7NZ + cat /tmp/tmp.gIPcXfzwGy + rm /tmp/tmp.26TkyTt7NZ /tmp/tmp.gIPcXfzwGy + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:52:40+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8PMNF7Dj0b +++ mktemp ++ local LAST_ERR=/tmp/tmp.GoFoZYUiRu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8PMNF7Dj0b ++ cat /tmp/tmp.GoFoZYUiRu ++ rm /tmp/tmp.8PMNF7Dj0b /tmp/tmp.GoFoZYUiRu ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.9FPx9c5fEd ++ mktemp + local LAST_ERR=/tmp/tmp.rUEDhEPscs + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9FPx9c5fEd + cat /tmp/tmp.rUEDhEPscs + rm /tmp/tmp.9FPx9c5fEd /tmp/tmp.rUEDhEPscs + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:52:42+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ltIanCS0Oi +++ mktemp ++ local LAST_ERR=/tmp/tmp.XxOH3CI8oh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ltIanCS0Oi ++ cat /tmp/tmp.XxOH3CI8oh ++ rm /tmp/tmp.ltIanCS0Oi /tmp/tmp.XxOH3CI8oh ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.WHC6MrDpwO ++ mktemp + local LAST_ERR=/tmp/tmp.ozYJbsjiY4 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WHC6MrDpwO + cat /tmp/tmp.ozYJbsjiY4 + rm /tmp/tmp.WHC6MrDpwO /tmp/tmp.ozYJbsjiY4 + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-08-04T19:51:33.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-08-04T19:51:03.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-08-04T19:52:10.000+0000 I listening for the commands + '[' -z '' ']' + desc 'Check GCS profiles' + set +o xtrace ----------------------------------------------------------------------------------- Check GCS profiles ----------------------------------------------------------------------------------- + compare_pbm_profile_setup some-name gcp-cs-s3 + local cluster=some-name + local profile=gcp-cs-s3 + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3 ++ mktemp + local LAST_OUT=/tmp/tmp.kCSbGJJx9n ++ mktemp + local LAST_ERR=/tmp/tmp.BS10MOhDpu + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-s3') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-s3 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kCSbGJJx9n + cat /tmp/tmp.BS10MOhDpu + rm /tmp/tmp.kCSbGJJx9n /tmp/tmp.BS10MOhDpu + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-s3.yml /tmp/tmp.UFgFtUGm4s/pbm_profile_gcp-cs-s3.yml + compare_pbm_profile_setup some-name gcp-cs-sa + local cluster=some-name + local profile=gcp-cs-sa + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa ++ mktemp + local LAST_OUT=/tmp/tmp.KaQnB6veV4 ++ mktemp + local LAST_ERR=/tmp/tmp.ij0cxrRD84 + local exit_status=0 + local timeout=4 + args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'profile' 'show' 'gcp-cs-sa') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-rs0-0 -c backup-agent -- pbm profile show gcp-cs-sa + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KaQnB6veV4 + cat /tmp/tmp.ij0cxrRD84 + rm /tmp/tmp.KaQnB6veV4 /tmp/tmp.ij0cxrRD84 + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/pbm_profile_gcp-cs-sa.yml /tmp/tmp.UFgFtUGm4s/pbm_profile_gcp-cs-sa.yml + backup_name_minio=backup-minio + '[' -z '' ']' + backup_name_aws=backup-aws-s3 + backup_name_gcp_s3=backup-gcp-cs-s3 + backup_name_gcp_sa=backup-gcp-cs-sa + backup_name_azure=backup-azure-blob + backup_name_oss_native=backup-oss-native + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-08-04T19:52:52+0000] running backup backup-aws-s3 + yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.b1mnEcgWxk ++ mktemp + local LAST_ERR=/tmp/tmp.UsBU12tl39 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b1mnEcgWxk perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.UsBU12tl39 + rm /tmp/tmp.b1mnEcgWxk /tmp/tmp.UsBU12tl39 + return 0 + run_backup gcp-cs-s3 + local storage=gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local type=logical + log 'running backup backup-gcp-cs-s3' + set +o xtrace [2026-08-04T19:52:54+0000] running backup backup-gcp-cs-s3 + kubectl_bin apply -f - + yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml ++ mktemp + local LAST_OUT=/tmp/tmp.pXzfeA59Ha ++ mktemp + local LAST_ERR=/tmp/tmp.wsLy3dfNSg + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pXzfeA59Ha perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created + cat /tmp/tmp.wsLy3dfNSg + rm /tmp/tmp.pXzfeA59Ha /tmp/tmp.wsLy3dfNSg + return 0 + run_backup gcp-cs-sa + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=logical + log 'running backup backup-gcp-cs-sa' + set +o xtrace [2026-08-04T19:52:56+0000] running backup backup-gcp-cs-sa + yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kdKBUZhddG ++ mktemp + local LAST_ERR=/tmp/tmp.KUwDBdEfwL + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kdKBUZhddG perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.KUwDBdEfwL + rm /tmp/tmp.kdKBUZhddG /tmp/tmp.KUwDBdEfwL + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-08-04T19:52:58+0000] running backup backup-azure-blob + yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ndRt1Cpjrz ++ mktemp + local LAST_ERR=/tmp/tmp.3ynnXIXK0v + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ndRt1Cpjrz perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.3ynnXIXK0v + rm /tmp/tmp.ndRt1Cpjrz /tmp/tmp.3ynnXIXK0v + return 0 + run_backup oss-native + local storage=oss-native + local backup_name=backup-oss-native + local type=logical + log 'running backup backup-oss-native' + set +o xtrace [2026-08-04T19:53:01+0000] running backup backup-oss-native + yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.UaRDR0YQ5O ++ mktemp + local LAST_ERR=/tmp/tmp.tYa6Thl2Cv + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UaRDR0YQ5O perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created + cat /tmp/tmp.tYa6Thl2Cv + rm /tmp/tmp.UaRDR0YQ5O /tmp/tmp.tYa6Thl2Cv + return 0 + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-08-04T19:53:03+0000] running backup backup-minio + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.QoFOBhrcDR ++ mktemp + local LAST_ERR=/tmp/tmp.DCjCDTalPw + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QoFOBhrcDR perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.DCjCDTalPw + rm /tmp/tmp.QoFOBhrcDR /tmp/tmp.DCjCDTalPw + return 0 + '[' -z '' ']' + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state..OK + wait_backup backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-s3 to reach ready state..........................OK + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state........OK + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state..OK + wait_backup backup-oss-native + local backup_name=backup-oss-native + local target_state=ready + set +o xtrace waiting for backup-oss-native to reach ready state..OK + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state..OK + sleep 5 + '[' -z '' ']' + desc 'check backup and restore -- aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- aws-s3 ----------------------------------------------------------------------------------- ++ get_backup_dest backup-aws-s3 ++ local backup_name=backup-aws-s3 ++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.4qZ7pDHalQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.xscC5KwbYH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4qZ7pDHalQ ++ cat /tmp/tmp.xscC5KwbYH ++ rm /tmp/tmp.4qZ7pDHalQ /tmp/tmp.xscC5KwbYH ++ return 0 + backup_dest_aws=operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_aws_credentials + local secret_name=aws-s3-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-08-04T19:52:54Z + key=/rs0/myApp.test.gz + endpoint= + storage_name=aws-s3 + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n '' ]] + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-08-04T19:52:54Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z//rs0/myApp.test.gz found in aws-s3' Backup operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z//rs0/myApp.test.gz found in aws-s3 + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + run_recovery_check backup-aws-s3 some-name-rs0 + local backup=backup-aws-s3 + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.icIkrCKplc +++ mktemp ++ local LAST_ERR=/tmp/tmp.HmXKOMCva2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.icIkrCKplc ++ cat /tmp/tmp.HmXKOMCva2 ++ rm /tmp/tmp.icIkrCKplc /tmp/tmp.HmXKOMCva2 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.CRIxIAssJA ++ mktemp + local LAST_ERR=/tmp/tmp.TzNM9Q1TKd + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CRIxIAssJA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f0158840-caeb-4d23-aca0-bbf167c6ba79") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.TzNM9Q1TKd + rm /tmp/tmp.CRIxIAssJA /tmp/tmp.TzNM9Q1TKd + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:54:29+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ejO1HO7umc +++ mktemp ++ local LAST_ERR=/tmp/tmp.G1vdpBHPeO ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ejO1HO7umc ++ cat /tmp/tmp.G1vdpBHPeO ++ rm /tmp/tmp.ejO1HO7umc /tmp/tmp.G1vdpBHPeO ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.7XvPJdOYgD ++ mktemp + local LAST_ERR=/tmp/tmp.iwxcRCWENz + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7XvPJdOYgD + cat /tmp/tmp.iwxcRCWENz + rm /tmp/tmp.7XvPJdOYgD /tmp/tmp.iwxcRCWENz + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-aws-s3 + local backup_name=backup-aws-s3 + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-aws-s3' + set +o xtrace [2026-08-04T19:54:31+0000] running restore restore-backup-aws-s3 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.czWPakNxpd ++ mktemp + local LAST_ERR=/tmp/tmp.QJMg26ggyh + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.czWPakNxpd perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created + cat /tmp/tmp.QJMg26ggyh + rm /tmp/tmp.czWPakNxpd /tmp/tmp.QJMg26ggyh + return 0 + wait_restore backup-aws-s3 some-name + local backup_name=backup-aws-s3 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jgMRoPII6T +++ mktemp ++ local LAST_ERR=/tmp/tmp.qKme7n1by4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jgMRoPII6T ++ cat /tmp/tmp.qKme7n1by4 ++ rm /tmp/tmp.jgMRoPII6T /tmp/tmp.qKme7n1by4 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:54:59+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9ulAdiAxBY +++ mktemp ++ local LAST_ERR=/tmp/tmp.25fv5d49A4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9ulAdiAxBY ++ cat /tmp/tmp.25fv5d49A4 ++ rm /tmp/tmp.9ulAdiAxBY /tmp/tmp.25fv5d49A4 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.YVgOkqG3UQ ++ mktemp + local LAST_ERR=/tmp/tmp.JRb1Wp06T1 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YVgOkqG3UQ + cat /tmp/tmp.JRb1Wp06T1 + rm /tmp/tmp.YVgOkqG3UQ /tmp/tmp.JRb1Wp06T1 + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:02+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q5xeTdPUBY +++ mktemp ++ local LAST_ERR=/tmp/tmp.8VtLsa2gg3 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q5xeTdPUBY ++ cat /tmp/tmp.8VtLsa2gg3 ++ rm /tmp/tmp.Q5xeTdPUBY /tmp/tmp.8VtLsa2gg3 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.osx5mzQ7CL ++ mktemp + local LAST_ERR=/tmp/tmp.mD5FGeApJM + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.osx5mzQ7CL + cat /tmp/tmp.mD5FGeApJM + rm /tmp/tmp.osx5mzQ7CL /tmp/tmp.mD5FGeApJM + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:04+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rDkOoMLN8J +++ mktemp ++ local LAST_ERR=/tmp/tmp.D9BUT8qsTN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rDkOoMLN8J ++ cat /tmp/tmp.D9BUT8qsTN ++ rm /tmp/tmp.rDkOoMLN8J /tmp/tmp.D9BUT8qsTN ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.fboM6yoHIE ++ mktemp + local LAST_ERR=/tmp/tmp.9o3OiK4mUO + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fboM6yoHIE + cat /tmp/tmp.9o3OiK4mUO + rm /tmp/tmp.fboM6yoHIE /tmp/tmp.9o3OiK4mUO + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'check backup and restore -- gcp-cs-s3' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs-s3 ----------------------------------------------------------------------------------- ++ get_backup_dest backup-gcp-cs-s3 ++ local backup_name=backup-gcp-cs-s3 ++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.AgoQVMpbd7 +++ mktemp ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.dHlqy4Ty3j ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AgoQVMpbd7 ++ cat /tmp/tmp.dHlqy4Ty3j ++ rm /tmp/tmp.AgoQVMpbd7 /tmp/tmp.dHlqy4Ty3j ++ return 0 + backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz ++ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz + gsutil_err=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz found in gcp-cs + run_recovery_check backup-gcp-cs-s3 some-name-rs0 + local backup=backup-gcp-cs-s3 + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qlo48BxLOY +++ mktemp ++ local LAST_ERR=/tmp/tmp.mReV9yJQXR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Qlo48BxLOY ++ cat /tmp/tmp.mReV9yJQXR ++ rm /tmp/tmp.Qlo48BxLOY /tmp/tmp.mReV9yJQXR ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.1bn663BNZv ++ mktemp + local LAST_ERR=/tmp/tmp.zHuaOl5DYK + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1bn663BNZv Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0afd8270-1b81-439a-927b-43f2c0081ea7") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.zHuaOl5DYK + rm /tmp/tmp.1bn663BNZv /tmp/tmp.zHuaOl5DYK + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:14+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.41hiCQ4p5c +++ mktemp ++ local LAST_ERR=/tmp/tmp.NgyDVLx30q ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.41hiCQ4p5c ++ cat /tmp/tmp.NgyDVLx30q ++ rm /tmp/tmp.41hiCQ4p5c /tmp/tmp.NgyDVLx30q ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.qEX5sinpRE ++ mktemp + local LAST_ERR=/tmp/tmp.wdfWIIq8iZ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qEX5sinpRE + cat /tmp/tmp.wdfWIIq8iZ + rm /tmp/tmp.qEX5sinpRE /tmp/tmp.wdfWIIq8iZ + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-gcp-cs-s3' + set +o xtrace [2026-08-04T19:55:17+0000] running restore restore-backup-gcp-cs-s3 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-s3/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-s3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.PHlfjzaD1D ++ mktemp + local LAST_ERR=/tmp/tmp.xfehbpTJyQ + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PHlfjzaD1D perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-s3 created + cat /tmp/tmp.xfehbpTJyQ + rm /tmp/tmp.PHlfjzaD1D /tmp/tmp.xfehbpTJyQ + return 0 + wait_restore backup-gcp-cs-s3 some-name + local backup_name=backup-gcp-cs-s3 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-s3 object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-s3 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hnaizJW2oX +++ mktemp ++ local LAST_ERR=/tmp/tmp.CzvU1TK1Zi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hnaizJW2oX ++ cat /tmp/tmp.CzvU1TK1Zi ++ rm /tmp/tmp.hnaizJW2oX /tmp/tmp.CzvU1TK1Zi ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:44+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3Q1DbBjf2J +++ mktemp ++ local LAST_ERR=/tmp/tmp.e2GzJx3Afr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3Q1DbBjf2J ++ cat /tmp/tmp.e2GzJx3Afr ++ rm /tmp/tmp.3Q1DbBjf2J /tmp/tmp.e2GzJx3Afr ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.QrfzWJ0BnT ++ mktemp + local LAST_ERR=/tmp/tmp.xmKFfum7EB + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QrfzWJ0BnT + cat /tmp/tmp.xmKFfum7EB + rm /tmp/tmp.QrfzWJ0BnT /tmp/tmp.xmKFfum7EB + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:47+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iyXbM6Cm1Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.7V6HVeEQ6K ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iyXbM6Cm1Z ++ cat /tmp/tmp.7V6HVeEQ6K ++ rm /tmp/tmp.iyXbM6Cm1Z /tmp/tmp.7V6HVeEQ6K ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.q4HLDlvKnr ++ mktemp + local LAST_ERR=/tmp/tmp.fr1HITzHPJ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q4HLDlvKnr + cat /tmp/tmp.fr1HITzHPJ + rm /tmp/tmp.q4HLDlvKnr /tmp/tmp.fr1HITzHPJ + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:50+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xuWGdK21U3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cFW1k2UyM2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xuWGdK21U3 ++ cat /tmp/tmp.cFW1k2UyM2 ++ rm /tmp/tmp.xuWGdK21U3 /tmp/tmp.cFW1k2UyM2 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.oUe05i1VRu ++ mktemp + local LAST_ERR=/tmp/tmp.DtHe7dWc3N + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oUe05i1VRu + cat /tmp/tmp.DtHe7dWc3N + rm /tmp/tmp.oUe05i1VRu /tmp/tmp.DtHe7dWc3N + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'check backup and restore -- gcp-cs-sa' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs-sa ----------------------------------------------------------------------------------- ++ get_backup_dest backup-gcp-cs-sa ++ local backup_name=backup-gcp-cs-sa ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.q3MAAW2A0Q +++ mktemp ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.4UvAh5viR0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-sa' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q3MAAW2A0Q ++ cat /tmp/tmp.4UvAh5viR0 ++ rm /tmp/tmp.q3MAAW2A0Q /tmp/tmp.4UvAh5viR0 ++ return 0 + backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z + check_backup_existence_gcs operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz ++ gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz + gsutil_err=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz found in gcp-cs + run_recovery_check backup-gcp-cs-sa some-name-rs0 + local backup=backup-gcp-cs-sa + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jmyz8f87Mu +++ mktemp ++ local LAST_ERR=/tmp/tmp.ctKdJbpCF8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Jmyz8f87Mu ++ cat /tmp/tmp.ctKdJbpCF8 ++ rm /tmp/tmp.Jmyz8f87Mu /tmp/tmp.ctKdJbpCF8 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.o5EbaZcP9b ++ mktemp + local LAST_ERR=/tmp/tmp.raQxwF1wC8 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.o5EbaZcP9b Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("3359d934-3f4c-49c2-a27e-6523b0173c47") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.raQxwF1wC8 + rm /tmp/tmp.o5EbaZcP9b /tmp/tmp.raQxwF1wC8 + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:55:56+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ogCwWj79Ka +++ mktemp ++ local LAST_ERR=/tmp/tmp.kEf6QuEojG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ogCwWj79Ka ++ cat /tmp/tmp.kEf6QuEojG ++ rm /tmp/tmp.ogCwWj79Ka /tmp/tmp.kEf6QuEojG ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.7bdx4phH6Z ++ mktemp + local LAST_ERR=/tmp/tmp.upuUzS93Nh + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7bdx4phH6Z + cat /tmp/tmp.upuUzS93Nh + rm /tmp/tmp.7bdx4phH6Z /tmp/tmp.upuUzS93Nh + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-gcp-cs-sa' + set +o xtrace [2026-08-04T19:55:58+0000] running restore restore-backup-gcp-cs-sa + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs-sa/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs-sa/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ENCtItxNTd ++ mktemp + local LAST_ERR=/tmp/tmp.SIZHJcOS7M + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ENCtItxNTd perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sa created + cat /tmp/tmp.SIZHJcOS7M + rm /tmp/tmp.ENCtItxNTd /tmp/tmp.SIZHJcOS7M + return 0 + wait_restore backup-gcp-cs-sa some-name + local backup_name=backup-gcp-cs-sa + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs-sa object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs-sa to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RjHUVo1LCk +++ mktemp ++ local LAST_ERR=/tmp/tmp.BcNWsZ6V9i ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RjHUVo1LCk ++ cat /tmp/tmp.BcNWsZ6V9i ++ rm /tmp/tmp.RjHUVo1LCk /tmp/tmp.BcNWsZ6V9i ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:56:29+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KODj0uecqV +++ mktemp ++ local LAST_ERR=/tmp/tmp.bNBqmAeitu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KODj0uecqV ++ cat /tmp/tmp.bNBqmAeitu ++ rm /tmp/tmp.KODj0uecqV /tmp/tmp.bNBqmAeitu ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.LBV5SOd0A0 ++ mktemp + local LAST_ERR=/tmp/tmp.9d9K4lcY9i + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LBV5SOd0A0 + cat /tmp/tmp.9d9K4lcY9i + rm /tmp/tmp.LBV5SOd0A0 /tmp/tmp.9d9K4lcY9i + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:56:32+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c1eaE9ZuZH +++ mktemp ++ local LAST_ERR=/tmp/tmp.aI74o1cDFL ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c1eaE9ZuZH ++ cat /tmp/tmp.aI74o1cDFL ++ rm /tmp/tmp.c1eaE9ZuZH /tmp/tmp.aI74o1cDFL ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.dIwpqiAHEX ++ mktemp + local LAST_ERR=/tmp/tmp.lahUiMpQqZ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dIwpqiAHEX + cat /tmp/tmp.lahUiMpQqZ + rm /tmp/tmp.dIwpqiAHEX /tmp/tmp.lahUiMpQqZ + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:56:35+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oMyQIUelBl +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZwEcQ0MZB7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oMyQIUelBl ++ cat /tmp/tmp.ZwEcQ0MZB7 ++ rm /tmp/tmp.oMyQIUelBl /tmp/tmp.ZwEcQ0MZB7 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.IWfRCwrZyc ++ mktemp + local LAST_ERR=/tmp/tmp.dqimqnxBq4 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IWfRCwrZyc + cat /tmp/tmp.dqimqnxBq4 + rm /tmp/tmp.IWfRCwrZyc /tmp/tmp.dqimqnxBq4 + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'check backup and restore -- azure-blob' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- azure-blob ----------------------------------------------------------------------------------- ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.tsxAp9ltL2 ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.8JcSbCc2Wa ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tsxAp9ltL2 ++ cat /tmp/tmp.8JcSbCc2Wa ++ rm /tmp/tmp.tsxAp9ltL2 /tmp/tmp.8JcSbCc2Wa ++ return 0 + backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z + setup_azure_credentials + local secret_name=azure-secret + echo 'Setting up Azure credentials from K8s secret: azure-secret' Setting up Azure credentials from K8s secret: azure-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'Azure credentials configured successfully' Azure credentials configured successfully + check_backup_existence_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z /rs0/myApp.test.gz ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup/2026-08-04T19:53:16Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz found in azure-blob' Backup psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz found in azure-blob + run_recovery_check backup-azure-blob some-name-rs0 + local backup=backup-azure-blob + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QjBEm4dBJm +++ mktemp ++ local LAST_ERR=/tmp/tmp.moOiUMUID1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QjBEm4dBJm ++ cat /tmp/tmp.moOiUMUID1 ++ rm /tmp/tmp.QjBEm4dBJm /tmp/tmp.moOiUMUID1 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.yLOvYF648S ++ mktemp + local LAST_ERR=/tmp/tmp.VMqgttavQh + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yLOvYF648S Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("21939b51-971f-4348-aa9b-1995d69042e7") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.VMqgttavQh + rm /tmp/tmp.yLOvYF648S /tmp/tmp.VMqgttavQh + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:56:43+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XoGmhn5pJR +++ mktemp ++ local LAST_ERR=/tmp/tmp.uu9vtqDMSa ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XoGmhn5pJR ++ cat /tmp/tmp.uu9vtqDMSa ++ rm /tmp/tmp.XoGmhn5pJR /tmp/tmp.uu9vtqDMSa ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.bS6VOiEPpM ++ mktemp + local LAST_ERR=/tmp/tmp.8J6EMR23XE + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bS6VOiEPpM + cat /tmp/tmp.8J6EMR23XE + rm /tmp/tmp.bS6VOiEPpM /tmp/tmp.8J6EMR23XE + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-azure-blob + local backup_name=backup-azure-blob + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-azure-blob' + set +o xtrace [2026-08-04T19:56:46+0000] running restore restore-backup-azure-blob + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xm5VaOqLh8 ++ mktemp + local LAST_ERR=/tmp/tmp.jjHzIxtQjF + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xm5VaOqLh8 perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created + cat /tmp/tmp.jjHzIxtQjF + rm /tmp/tmp.xm5VaOqLh8 /tmp/tmp.jjHzIxtQjF + return 0 + wait_restore backup-azure-blob some-name + local backup_name=backup-azure-blob + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VTqnG2YiN2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ry04l0BLX2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VTqnG2YiN2 ++ cat /tmp/tmp.ry04l0BLX2 ++ rm /tmp/tmp.VTqnG2YiN2 /tmp/tmp.ry04l0BLX2 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:57:14+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eoksQRVvxV +++ mktemp ++ local LAST_ERR=/tmp/tmp.B01V79Ot6O ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eoksQRVvxV ++ cat /tmp/tmp.B01V79Ot6O ++ rm /tmp/tmp.eoksQRVvxV /tmp/tmp.B01V79Ot6O ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.6TxkgJW795 ++ mktemp + local LAST_ERR=/tmp/tmp.vJc55B1AOv + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6TxkgJW795 + cat /tmp/tmp.vJc55B1AOv + rm /tmp/tmp.6TxkgJW795 /tmp/tmp.vJc55B1AOv + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:57:16+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vBNlObZsGG +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZR73HT8aNq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vBNlObZsGG ++ cat /tmp/tmp.ZR73HT8aNq ++ rm /tmp/tmp.vBNlObZsGG /tmp/tmp.ZR73HT8aNq ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.AwWhClNH0P ++ mktemp + local LAST_ERR=/tmp/tmp.K0t6rToSyC + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AwWhClNH0P + cat /tmp/tmp.K0t6rToSyC + rm /tmp/tmp.AwWhClNH0P /tmp/tmp.K0t6rToSyC + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:57:19+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FRhNMPuFAr +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZtUybCWmVS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FRhNMPuFAr ++ cat /tmp/tmp.ZtUybCWmVS ++ rm /tmp/tmp.FRhNMPuFAr /tmp/tmp.ZtUybCWmVS ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.x8GaDAl67i ++ mktemp + local LAST_ERR=/tmp/tmp.rgVZMQ3BnK + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.x8GaDAl67i + cat /tmp/tmp.rgVZMQ3BnK + rm /tmp/tmp.x8GaDAl67i /tmp/tmp.rgVZMQ3BnK + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'check backup and restore -- oss-native' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- oss-native ----------------------------------------------------------------------------------- ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.3ibAnfsJ95 ++ sed 's|gs://||' +++ mktemp ++ sed 's|oss://||' ++ local LAST_ERR=/tmp/tmp.41xza4lCRb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3ibAnfsJ95 ++ cat /tmp/tmp.41xza4lCRb ++ rm /tmp/tmp.3ibAnfsJ95 /tmp/tmp.41xza4lCRb ++ return 0 + backup_dest_oss_native=operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vLbb8LZ7KT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oWph7U2GPI +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vLbb8LZ7KT +++ cat /tmp/tmp.oWph7U2GPI +++ rm /tmp/tmp.vLbb8LZ7KT /tmp/tmp.oWph7U2GPI +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6GbsSuZHDb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yg9vyRjswG +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.6GbsSuZHDb +++ cat /tmp/tmp.yg9vyRjswG +++ rm /tmp/tmp.6GbsSuZHDb /tmp/tmp.yg9vyRjswG +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7G9McoYtXb +++ mktemp ++ local LAST_ERR=/tmp/tmp.7w1cmGQdBA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7G9McoYtXb ++ cat /tmp/tmp.7w1cmGQdBA ++ rm /tmp/tmp.7G9McoYtXb /tmp/tmp.7w1cmGQdBA ++ return 0 + backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gEsjTWeYOH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cEyki4fjqC +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.gEsjTWeYOH +++ cat /tmp/tmp.cEyki4fjqC +++ rm /tmp/tmp.gEsjTWeYOH /tmp/tmp.cEyki4fjqC +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OC0QCLEAHX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iMlLY4DB57 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OC0QCLEAHX +++ cat /tmp/tmp.iMlLY4DB57 +++ rm /tmp/tmp.OC0QCLEAHX /tmp/tmp.iMlLY4DB57 +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YTYEM6RgfV +++ mktemp ++ local LAST_ERR=/tmp/tmp.EQISdlNifQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YTYEM6RgfV ++ cat /tmp/tmp.EQISdlNifQ ++ rm /tmp/tmp.YTYEM6RgfV /tmp/tmp.EQISdlNifQ ++ return 0 + backup_region_oss_native=eu-central-1 + check_backup_in_storage backup-oss-native oss rs0 myApp.test.gz + local backup=backup-oss-native + local storage_type=oss + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.9UjHN6dreH +++ mktemp ++ local LAST_ERR=/tmp/tmp.hPQAhGV91h ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9UjHN6dreH ++ cat /tmp/tmp.hPQAhGV91h ++ rm /tmp/tmp.9UjHN6dreH /tmp/tmp.hPQAhGV91h ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GJRZWwhKBh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.o0WEYFhvjl +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GJRZWwhKBh +++ cat /tmp/tmp.o0WEYFhvjl +++ rm /tmp/tmp.GJRZWwhKBh /tmp/tmp.o0WEYFhvjl +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.M9DeA9qYbI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.949HbdVF44 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.M9DeA9qYbI +++ cat /tmp/tmp.949HbdVF44 +++ rm /tmp/tmp.M9DeA9qYbI /tmp/tmp.949HbdVF44 +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WKwJFzlMCy +++ mktemp ++ local LAST_ERR=/tmp/tmp.VwIRbY2LBC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WKwJFzlMCy ++ cat /tmp/tmp.VwIRbY2LBC ++ rm /tmp/tmp.WKwJFzlMCy /tmp/tmp.VwIRbY2LBC ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ADoa8nsSJv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dkc0PCOhEi +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ADoa8nsSJv +++ cat /tmp/tmp.dkc0PCOhEi +++ rm /tmp/tmp.ADoa8nsSJv /tmp/tmp.dkc0PCOhEi +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lHdrn525yy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LffOSIlLQG +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lHdrn525yy +++ cat /tmp/tmp.LffOSIlLQG +++ rm /tmp/tmp.lHdrn525yy /tmp/tmp.LffOSIlLQG +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.m9EauUaUGb +++ mktemp ++ local LAST_ERR=/tmp/tmp.dRfD2HLRof ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.m9EauUaUGb ++ cat /tmp/tmp.dRfD2HLRof ++ rm /tmp/tmp.m9EauUaUGb /tmp/tmp.dRfD2HLRof ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z /rs0/myApp.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z + key=/rs0/myApp.test.gz + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z//rs0/myApp.test.gz found in oss-native' Backup operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z//rs0/myApp.test.gz found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + run_recovery_check backup-oss-native some-name-rs0 + local backup=backup-oss-native + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7dV7HLfJga +++ mktemp ++ local LAST_ERR=/tmp/tmp.97crm8gqH6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7dV7HLfJga ++ cat /tmp/tmp.97crm8gqH6 ++ rm /tmp/tmp.7dV7HLfJga /tmp/tmp.97crm8gqH6 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.koHAvAnL3J ++ mktemp + local LAST_ERR=/tmp/tmp.250T4y5D3t + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.koHAvAnL3J Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("9d9f89f3-ddd8-41c4-88bf-5a08870d8d0d") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.250T4y5D3t + rm /tmp/tmp.koHAvAnL3J /tmp/tmp.250T4y5D3t + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:57:41+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.om6wTi6bxe +++ mktemp ++ local LAST_ERR=/tmp/tmp.DGMBLCrSOK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.om6wTi6bxe ++ cat /tmp/tmp.DGMBLCrSOK ++ rm /tmp/tmp.om6wTi6bxe /tmp/tmp.DGMBLCrSOK ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.mW7x1LQEpg ++ mktemp + local LAST_ERR=/tmp/tmp.TG1JyP1l7H + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mW7x1LQEpg + cat /tmp/tmp.TG1JyP1l7H + rm /tmp/tmp.mW7x1LQEpg /tmp/tmp.TG1JyP1l7H + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-oss-native + local backup_name=backup-oss-native + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-oss-native' + set +o xtrace [2026-08-04T19:57:44+0000] running restore restore-backup-oss-native + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-oss-native/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-oss-native/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.gQJDNKptUw ++ mktemp + local LAST_ERR=/tmp/tmp.6UhfmExadK + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gQJDNKptUw perconaservermongodbrestore.psmdb.percona.com/restore-backup-oss-native created + cat /tmp/tmp.6UhfmExadK + rm /tmp/tmp.gQJDNKptUw /tmp/tmp.6UhfmExadK + return 0 + wait_restore backup-oss-native some-name + local backup_name=backup-oss-native + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-oss-native object to be created.OK Waiting psmdb-restore/restore-backup-oss-native to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DNeUbMLBZx +++ mktemp ++ local LAST_ERR=/tmp/tmp.k24jO6H2jN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DNeUbMLBZx ++ cat /tmp/tmp.k24jO6H2jN ++ rm /tmp/tmp.DNeUbMLBZx /tmp/tmp.k24jO6H2jN ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:58:18+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DgGkywgLd1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tk1wygMVi8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DgGkywgLd1 ++ cat /tmp/tmp.tk1wygMVi8 ++ rm /tmp/tmp.DgGkywgLd1 /tmp/tmp.tk1wygMVi8 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.SSZUNWi8An ++ mktemp + local LAST_ERR=/tmp/tmp.1SFfaY8FmG + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SSZUNWi8An + cat /tmp/tmp.1SFfaY8FmG + rm /tmp/tmp.SSZUNWi8An /tmp/tmp.1SFfaY8FmG + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:58:20+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5RyZT6u51Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.3PbOX4Rfpr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5RyZT6u51Y ++ cat /tmp/tmp.3PbOX4Rfpr ++ rm /tmp/tmp.5RyZT6u51Y /tmp/tmp.3PbOX4Rfpr ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.pVizIzds1X ++ mktemp + local LAST_ERR=/tmp/tmp.yIaOnFyCOQ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pVizIzds1X + cat /tmp/tmp.yIaOnFyCOQ + rm /tmp/tmp.pVizIzds1X /tmp/tmp.yIaOnFyCOQ + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:58:23+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.89w24hospl +++ mktemp ++ local LAST_ERR=/tmp/tmp.pyOYdJ49pv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.89w24hospl ++ cat /tmp/tmp.pyOYdJ49pv ++ rm /tmp/tmp.89w24hospl /tmp/tmp.pyOYdJ49pv ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.iHnGIsvg2A ++ mktemp + local LAST_ERR=/tmp/tmp.tohdV66kGH + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iHnGIsvg2A + cat /tmp/tmp.tohdV66kGH + rm /tmp/tmp.iHnGIsvg2A /tmp/tmp.tohdV66kGH + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'check backup and restore -- minio' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- minio ----------------------------------------------------------------------------------- + check_backup_in_storage backup-minio minio rs0 myApp.test.gz + local backup=backup-minio + local storage_type=minio + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-minio ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-minio ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' +++ mktemp ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ local LAST_OUT=/tmp/tmp.Ox5Uoyp5Gy ++ sed 's|gs://||' ++ sed 's|oss://||' +++ mktemp ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.RLW2rnTk35 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ox5Uoyp5Gy ++ cat /tmp/tmp.RLW2rnTk35 ++ rm /tmp/tmp.Ox5Uoyp5Gy /tmp/tmp.RLW2rnTk35 ++ return 0 + backup_dest=operator-testing/2026-08-04T19:53:05Z + case ${storage_type} in + endpoint=minio-service + aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:53:05Z/rs0/myApp.test.gz + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:53:05Z/rs0/myApp.test.gz + grep myApp.test.gz ++ mktemp + local LAST_OUT=/tmp/tmp.I1nSJePH1e ++ mktemp + local LAST_ERR=/tmp/tmp.p4AisclUOX + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/2026-08-04T19:53:05Z/rs0/myApp.test.gz') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:53:05Z/rs0/myApp.test.gz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I1nSJePH1e + cat /tmp/tmp.p4AisclUOX All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.I1nSJePH1e /tmp/tmp.p4AisclUOX + return 0 2026-08-04 19:53:09 55 myApp.test.gz + run_recovery_check backup-minio some-name-rs0 + local backup=backup-minio + local cluster=some-name-rs0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5NXENllHj4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aL3SwgGPLH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5NXENllHj4 ++ cat /tmp/tmp.aL3SwgGPLH ++ rm /tmp/tmp.5NXENllHj4 /tmp/tmp.aL3SwgGPLH ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.2ihXnpBez6 ++ mktemp + local LAST_ERR=/tmp/tmp.V4Z6cUe201 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2ihXnpBez6 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("981f0a70-3440-4d3d-aae0-aed5d20b1e62") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.V4Z6cUe201 + rm /tmp/tmp.2ihXnpBez6 /tmp/tmp.V4Z6cUe201 + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:58:33+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8fJ4NnXVEW +++ mktemp ++ local LAST_ERR=/tmp/tmp.eLoqS6mJPY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8fJ4NnXVEW ++ cat /tmp/tmp.eLoqS6mJPY ++ rm /tmp/tmp.8fJ4NnXVEW /tmp/tmp.eLoqS6mJPY ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.eZZmntA6ro ++ mktemp + local LAST_ERR=/tmp/tmp.XNDQq2TAFX + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eZZmntA6ro + cat /tmp/tmp.XNDQq2TAFX + rm /tmp/tmp.eZZmntA6ro /tmp/tmp.XNDQq2TAFX + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore backup-minio + local backup_name=backup-minio + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + log 'running restore restore-backup-minio' + set +o xtrace [2026-08-04T19:58:36+0000] running restore restore-backup-minio + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.p5IgprwcCU ++ mktemp + local LAST_ERR=/tmp/tmp.KvdL8aFkPC + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p5IgprwcCU perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created + cat /tmp/tmp.KvdL8aFkPC + rm /tmp/tmp.p5IgprwcCU /tmp/tmp.KvdL8aFkPC + return 0 + wait_restore backup-minio some-name + local backup_name=backup-minio + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio object to be created.OK Waiting psmdb-restore/restore-backup-minio to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U0ik8JAmPN +++ mktemp ++ local LAST_ERR=/tmp/tmp.0ol1dMe5oy ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U0ik8JAmPN ++ cat /tmp/tmp.0ol1dMe5oy ++ rm /tmp/tmp.U0ik8JAmPN /tmp/tmp.0ol1dMe5oy ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:59:03+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7bTOdczhEl +++ mktemp ++ local LAST_ERR=/tmp/tmp.8PjEghUHRM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7bTOdczhEl ++ cat /tmp/tmp.8PjEghUHRM ++ rm /tmp/tmp.7bTOdczhEl /tmp/tmp.8PjEghUHRM ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.B4jHd3GBLj ++ mktemp + local LAST_ERR=/tmp/tmp.03PCgOYb97 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.B4jHd3GBLj + cat /tmp/tmp.03PCgOYb97 + rm /tmp/tmp.B4jHd3GBLj /tmp/tmp.03PCgOYb97 + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:59:06+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J8GHmlWxyj +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z3lAJwm6y0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J8GHmlWxyj ++ cat /tmp/tmp.Z3lAJwm6y0 ++ rm /tmp/tmp.J8GHmlWxyj /tmp/tmp.Z3lAJwm6y0 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.nP2O4fxlDx ++ mktemp + local LAST_ERR=/tmp/tmp.QJJCti2nxb + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nP2O4fxlDx + cat /tmp/tmp.QJJCti2nxb + rm /tmp/tmp.nP2O4fxlDx /tmp/tmp.QJJCti2nxb + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T19:59:08+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5yC4KEcwYB +++ mktemp ++ local LAST_ERR=/tmp/tmp.TFsQgZZZz2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5yC4KEcwYB ++ cat /tmp/tmp.TFsQgZZZz2 ++ rm /tmp/tmp.5yC4KEcwYB /tmp/tmp.TFsQgZZZz2 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.nrJP6XpMvf ++ mktemp + local LAST_ERR=/tmp/tmp.mpjufJ3B9V + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nrJP6XpMvf + cat /tmp/tmp.mpjufJ3B9V + rm /tmp/tmp.nrJP6XpMvf /tmp/tmp.mpjufJ3B9V + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + run_mongo 'use myApp\n db.dropUser("test1user")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.dropUser("test1user")' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LvANMvdFCF +++ mktemp ++ local LAST_ERR=/tmp/tmp.f2w1fw67zO ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LvANMvdFCF ++ cat /tmp/tmp.f2w1fw67zO ++ rm /tmp/tmp.LvANMvdFCF /tmp/tmp.f2w1fw67zO ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.7wRQzS05bN ++ mktemp + local LAST_ERR=/tmp/tmp.y4h1fF56hz + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropUser("test1user")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7wRQzS05bN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("e14aeda6-ac8e-49f1-b7c7-e424e8b5d051") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.y4h1fF56hz + rm /tmp/tmp.7wRQzS05bN /tmp/tmp.y4h1fF56hz + return 0 + run_mongo 'use myApp\n db.dropRole("test1role")' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.dropRole("test1role")' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RjNK5wNPVw +++ mktemp ++ local LAST_ERR=/tmp/tmp.rtL9LgtQ3x ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RjNK5wNPVw ++ cat /tmp/tmp.rtL9LgtQ3x ++ rm /tmp/tmp.RjNK5wNPVw /tmp/tmp.rtL9LgtQ3x ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.WsiO2k6Bqn ++ mktemp + local LAST_ERR=/tmp/tmp.dAtUVkQPpq + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.dropRole("test1role")\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WsiO2k6Bqn Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("f794796f-1457-4a19-a9a8-b99826cd749b") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.dAtUVkQPpq + rm /tmp/tmp.WsiO2k6Bqn /tmp/tmp.dAtUVkQPpq + return 0 + desc 'selective restore -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore -- minio ----------------------------------------------------------------------------------- + run_recovery_check_selective backup-minio some-name-rs0 myApp test2 false test1user test1role + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local collection=test2 + local with_users_and_roles=false + local username=test1user + local role=test1role + restore_name=restore-backup-minio-selective + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uguGcscw44 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZFlZap4VOj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uguGcscw44 ++ cat /tmp/tmp.ZFlZap4VOj ++ rm /tmp/tmp.uguGcscw44 /tmp/tmp.ZFlZap4VOj ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.T5awNN2o76 ++ mktemp + local LAST_ERR=/tmp/tmp.QLo4PPkYPU + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T5awNN2o76 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("da977b4e-54ac-407d-bca8-3bdd047a70f3") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.QLo4PPkYPU + rm /tmp/tmp.T5awNN2o76 /tmp/tmp.QLo4PPkYPU + return 0 ++ collection_exists test2 ++ local collection=test2 ++ grep -v 'switched to' ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aH6CeHHynP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.I77U1DfEM0 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.aH6CeHHynP +++ cat /tmp/tmp.I77U1DfEM0 +++ rm /tmp/tmp.aH6CeHHynP /tmp/tmp.I77U1DfEM0 +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K1JV29NUdt +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZtpMr5RUEH ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K1JV29NUdt ++ cat /tmp/tmp.ZtpMr5RUEH ++ rm /tmp/tmp.K1JV29NUdt /tmp/tmp.ZtpMr5RUEH ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + yq '.metadata.name="restore-backup-minio-selective"' + yq '.spec.backupName="backup-minio"' + yq '.spec.selective.namespaces[0]="myApp.test"' + yq .spec.selective.withUsersAndRoles=false + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.3oLZZhJVe5 ++ mktemp + local LAST_ERR=/tmp/tmp.Wos78uSUlm + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3oLZZhJVe5 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created + cat /tmp/tmp.Wos78uSUlm + rm /tmp/tmp.3oLZZhJVe5 /tmp/tmp.Wos78uSUlm + return 0 + wait_restore backup-minio-selective some-name + local backup_name=backup-minio-selective + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GfSwxlkPGP +++ mktemp ++ local LAST_ERR=/tmp/tmp.jtqBNps3nC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GfSwxlkPGP ++ cat /tmp/tmp.jtqBNps3nC ++ rm /tmp/tmp.GfSwxlkPGP /tmp/tmp.jtqBNps3nC ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ grep -v 'switched to' ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qx3PlukVLb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OXbdf8zzrr +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qx3PlukVLb +++ cat /tmp/tmp.OXbdf8zzrr +++ rm /tmp/tmp.qx3PlukVLb /tmp/tmp.OXbdf8zzrr +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9YHZIO4wxJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.kTCUx0yhFe ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9YHZIO4wxJ ++ cat /tmp/tmp.kTCUx0yhFe ++ rm /tmp/tmp.9YHZIO4wxJ /tmp/tmp.kTCUx0yhFe ++ return 0 + [[ false == true ]] + [[ false == true ]] + [[ false == false ]] ++ user_exists test1user ++ local username=test1user ++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 '' '' --quiet ++ grep -v 'switched to' ++ local 'command=use myApp\n JSON.stringify(db.getUsers())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'any(.[]; ._id==myApp.test1user)' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AL7FChr8hH ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error +++ local LAST_ERR=/tmp/tmp.VIhd5C5c0R +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.AL7FChr8hH +++ cat /tmp/tmp.VIhd5C5c0R +++ rm /tmp/tmp.AL7FChr8hH /tmp/tmp.VIhd5C5c0R +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V6vfGNsAY0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Hc7Sgr18xW ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V6vfGNsAY0 ++ cat /tmp/tmp.Hc7Sgr18xW ++ rm /tmp/tmp.V6vfGNsAY0 /tmp/tmp.Hc7Sgr18xW ++ return 0 + [[ '' == true ]] ++ role_exists test1role ++ local role=test1role ++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 '' '' --quiet ++ grep -v 'switched to' ++ jq 'any(.[]; ._id==myApp.test1role)' ++ local 'command=use myApp\n JSON.stringify(db.getRoles())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.e4wWFBIVxx jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Af8ix0XRTn +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.e4wWFBIVxx +++ cat /tmp/tmp.Af8ix0XRTn +++ rm /tmp/tmp.e4wWFBIVxx /tmp/tmp.Af8ix0XRTn +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HaV8dd7NFN +++ mktemp ++ local LAST_ERR=/tmp/tmp.9P3sh4fkNW ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HaV8dd7NFN ++ cat /tmp/tmp.9P3sh4fkNW ++ rm /tmp/tmp.HaV8dd7NFN /tmp/tmp.9P3sh4fkNW ++ return 0 + [[ '' == true ]] + kubectl_bin delete psmdb-restore restore-backup-minio-selective ++ mktemp + local LAST_OUT=/tmp/tmp.A24Iu8oNFu ++ mktemp + local LAST_ERR=/tmp/tmp.95dxyVFwyy + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-restore' 'restore-backup-minio-selective') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-restore restore-backup-minio-selective + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A24Iu8oNFu perconaservermongodbrestore.psmdb.percona.com "restore-backup-minio-selective" deleted from demand-backup-26283 namespace + cat /tmp/tmp.95dxyVFwyy + rm /tmp/tmp.A24Iu8oNFu /tmp/tmp.95dxyVFwyy + return 0 + run_mongo 'use myApp\n db.test2.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test2.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JGSjpeXQjr +++ mktemp ++ local LAST_ERR=/tmp/tmp.gPBRTq5A3j ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JGSjpeXQjr ++ cat /tmp/tmp.gPBRTq5A3j ++ rm /tmp/tmp.JGSjpeXQjr /tmp/tmp.gPBRTq5A3j ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.J1yhSWfnsU ++ mktemp + local LAST_ERR=/tmp/tmp.QsOMxxMRKO + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J1yhSWfnsU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("1749fdae-8336-420f-92f1-e0205712557c") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.QsOMxxMRKO + rm /tmp/tmp.J1yhSWfnsU /tmp/tmp.QsOMxxMRKO + return 0 + desc 'selective restore with users and roles -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore with users and roles -- minio ----------------------------------------------------------------------------------- + run_recovery_check_selective backup-minio some-name-rs0 myApp test2 true test1user test1role + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local collection=test2 + local with_users_and_roles=true + local username=test1user + local role=test1role + restore_name=restore-backup-minio-selective + run_mongo 'use myApp\n db.test2.drop()' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test2.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.laJKmgFfJp +++ mktemp ++ local LAST_ERR=/tmp/tmp.pYZYVi6Vyq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.laJKmgFfJp ++ cat /tmp/tmp.pYZYVi6Vyq ++ rm /tmp/tmp.laJKmgFfJp /tmp/tmp.pYZYVi6Vyq ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.aNB8fNI7h1 ++ mktemp + local LAST_ERR=/tmp/tmp.qheXpwQxpT + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test2.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aNB8fNI7h1 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("562fb39b-5435-4e58-9791-5361c9ed211b") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.qheXpwQxpT + rm /tmp/tmp.aNB8fNI7h1 /tmp/tmp.qheXpwQxpT + return 0 ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ grep -v 'switched to' ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ruaULwSjvT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KIX3if72gY +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ruaULwSjvT +++ cat /tmp/tmp.KIX3if72gY +++ rm /tmp/tmp.ruaULwSjvT /tmp/tmp.KIX3if72gY +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LzOr6OEued +++ mktemp ++ local LAST_ERR=/tmp/tmp.Y4iinmnKls ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LzOr6OEued ++ cat /tmp/tmp.Y4iinmnKls ++ rm /tmp/tmp.LzOr6OEued /tmp/tmp.Y4iinmnKls ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + yq '.metadata.name="restore-backup-minio-selective"' + yq '.spec.backupName="backup-minio"' + yq '.spec.selective.namespaces[0]="myApp.test"' + yq .spec.selective.withUsersAndRoles=true + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.LJkLryxK6C ++ mktemp + local LAST_ERR=/tmp/tmp.ey72Fy1Ddf + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LJkLryxK6C perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-selective created + cat /tmp/tmp.ey72Fy1Ddf + rm /tmp/tmp.LJkLryxK6C /tmp/tmp.ey72Fy1Ddf + return 0 + wait_restore backup-minio-selective some-name + local backup_name=backup-minio-selective + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-selective object to be created.OK Waiting psmdb-restore/restore-backup-minio-selective to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gUdJNHiAcB +++ mktemp ++ local LAST_ERR=/tmp/tmp.5rcBTJLrlz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gUdJNHiAcB ++ cat /tmp/tmp.5rcBTJLrlz ++ rm /tmp/tmp.gUdJNHiAcB /tmp/tmp.5rcBTJLrlz ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test2 ++ local collection=test2 ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ grep -v 'switched to' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'index("test2") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3PUcN8IJdA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5h6GIINLKs +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3PUcN8IJdA +++ cat /tmp/tmp.5h6GIINLKs +++ rm /tmp/tmp.3PUcN8IJdA /tmp/tmp.5h6GIINLKs +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OXIcSnL52w +++ mktemp ++ local LAST_ERR=/tmp/tmp.3HjAqydzl6 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OXIcSnL52w ++ cat /tmp/tmp.3HjAqydzl6 ++ rm /tmp/tmp.OXIcSnL52w /tmp/tmp.3HjAqydzl6 ++ return 0 + [[ false == true ]] + [[ true == true ]] ++ user_exists test1user ++ local username=test1user ++ run_mongo 'use myApp\n JSON.stringify(db.getUsers())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getUsers())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ jq 'any(.[]; ._id==myApp.test1user)' ++ grep -v 'switched to' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1user) jq: 1 compile error +++ local LAST_OUT=/tmp/tmp.o2EkkhVsOO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HQL1Z1nuKO +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.o2EkkhVsOO +++ cat /tmp/tmp.HQL1Z1nuKO +++ rm /tmp/tmp.o2EkkhVsOO /tmp/tmp.HQL1Z1nuKO +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O6jQ9zSFe8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xWfsuT1rKP ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getUsers())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.O6jQ9zSFe8 ++ cat /tmp/tmp.xWfsuT1rKP ++ rm /tmp/tmp.O6jQ9zSFe8 /tmp/tmp.xWfsuT1rKP ++ return 0 + [[ '' == false ]] ++ role_exists test1role ++ local role=test1role ++ grep -v 'switched to' ++ run_mongo 'use myApp\n JSON.stringify(db.getRoles())' userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 '' '' --quiet ++ jq 'any(.[]; ._id==myApp.test1role)' ++ local 'command=use myApp\n JSON.stringify(db.getRoles())' ++ local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp jq: error: myApp/0 is not defined at , line 1: any(.[]; ._id==myApp.test1role) jq: 1 compile error +++ local LAST_OUT=/tmp/tmp.dDxVWUzYUe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cZWqsjpml8 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dDxVWUzYUe +++ cat /tmp/tmp.cZWqsjpml8 +++ rm /tmp/tmp.dDxVWUzYUe /tmp/tmp.cZWqsjpml8 +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qvOGkBxFJa +++ mktemp ++ local LAST_ERR=/tmp/tmp.P91WiiuvNP ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getRoles())\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qvOGkBxFJa ++ cat /tmp/tmp.P91WiiuvNP ++ rm /tmp/tmp.qvOGkBxFJa /tmp/tmp.P91WiiuvNP ++ return 0 + [[ '' == false ]] + desc 'selective restore with namespace remap -- minio' + set +o xtrace ----------------------------------------------------------------------------------- selective restore with namespace remap -- minio ----------------------------------------------------------------------------------- + run_recovery_check_namespace_remap backup-minio some-name-rs0 myApp test test_remapped + local backup=backup-minio + local cluster=some-name-rs0 + local database=myApp + local src_collection=test + local dst_collection=test_remapped + restore_name=restore-backup-minio-remapped + run_mongo 'use myApp\n db.test_remapped.drop()' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test_remapped.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ytye1Jvzrb +++ mktemp ++ local LAST_ERR=/tmp/tmp.1xcU7cLH12 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ytye1Jvzrb ++ cat /tmp/tmp.1xcU7cLH12 ++ rm /tmp/tmp.Ytye1Jvzrb /tmp/tmp.1xcU7cLH12 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.tjQ1mmtwMg ++ mktemp + local LAST_ERR=/tmp/tmp.COTpfOJHDE + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tjQ1mmtwMg Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ecb9cabc-6919-400a-bbca-bf217efc19d7") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.COTpfOJHDE + rm /tmp/tmp.tjQ1mmtwMg /tmp/tmp.COTpfOJHDE + return 0 ++ collection_exists test_remapped ++ local collection=test_remapped ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ grep -v 'switched to' ++ jq 'index("test_remapped") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kCWVDNGxWs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XWJmu3syLP +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.kCWVDNGxWs +++ cat /tmp/tmp.XWJmu3syLP +++ rm /tmp/tmp.kCWVDNGxWs /tmp/tmp.XWJmu3syLP +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TEBKvjWul1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.D2rIeCNOQf ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TEBKvjWul1 ++ cat /tmp/tmp.D2rIeCNOQf ++ rm /tmp/tmp.TEBKvjWul1 /tmp/tmp.D2rIeCNOQf ++ return 0 + [[ false == true ]] + yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore.yml + yq '.spec.backupName="backup-minio"' + yq '.metadata.name="restore-backup-minio-remapped"' + yq '.spec.selective.nsTo="myApp.test_remapped"' + yq '.spec.selective.nsFrom="myApp.test"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.JdmaK4W7eI ++ mktemp + local LAST_ERR=/tmp/tmp.tTAKcfdm1J + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JdmaK4W7eI perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-remapped created + cat /tmp/tmp.tTAKcfdm1J + rm /tmp/tmp.JdmaK4W7eI /tmp/tmp.tTAKcfdm1J + return 0 + wait_restore backup-minio-remapped some-name + local backup_name=backup-minio-remapped + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-remapped object to be created.OK Waiting psmdb-restore/restore-backup-minio-remapped to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yFbbmvDPZP +++ mktemp ++ local LAST_ERR=/tmp/tmp.32iltbGFJD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yFbbmvDPZP ++ cat /tmp/tmp.32iltbGFJD ++ rm /tmp/tmp.yFbbmvDPZP /tmp/tmp.32iltbGFJD ++ return 0 + [[ ready == ready ]] + echo .OK .OK ++ collection_exists test_remapped ++ local collection=test_remapped ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ grep -v 'switched to' ++ jq 'index("test_remapped") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iVqO6LePk5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.notuAfunlv +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.iVqO6LePk5 +++ cat /tmp/tmp.notuAfunlv +++ rm /tmp/tmp.iVqO6LePk5 /tmp/tmp.notuAfunlv +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2EIxmVIKxC +++ mktemp ++ local LAST_ERR=/tmp/tmp.JMNLkVCiul ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2EIxmVIKxC ++ cat /tmp/tmp.JMNLkVCiul ++ rm /tmp/tmp.2EIxmVIKxC /tmp/tmp.JMNLkVCiul ++ return 0 + [[ true == false ]] ++ collection_exists test ++ local collection=test ++ run_mongo 'use myApp\n JSON.stringify(db.getCollectionNames())' myApp:myPass@some-name-rs0.demand-backup-26283 '' '' --quiet ++ local 'command=use myApp\n JSON.stringify(db.getCollectionNames())' ++ local uri=myApp:myPass@some-name-rs0.demand-backup-26283 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local replica_set=rs0 ++ [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ grep -v 'switched to' ++ jq 'index("test") != null' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UTgWsAYRzK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XzAOFwQEiN +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UTgWsAYRzK +++ cat /tmp/tmp.XzAOFwQEiN +++ rm /tmp/tmp.UTgWsAYRzK /tmp/tmp.XzAOFwQEiN +++ return 0 ++ local client_container=psmdb-client-757cd65695-k2wrh ++ kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YsXhaHBS98 +++ mktemp ++ local LAST_ERR=/tmp/tmp.GaeoWI1BQ5 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n JSON.stringify(db.getCollectionNames())\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YsXhaHBS98 ++ cat /tmp/tmp.GaeoWI1BQ5 ++ rm /tmp/tmp.YsXhaHBS98 /tmp/tmp.GaeoWI1BQ5 ++ return 0 + [[ true == false ]] + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -remapped .svc.cluster.local myApp test_remapped + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-remapped + local suffix=.svc.cluster.local + local database=myApp + local collection=test_remapped + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test_remapped.find()' + [[ -n '' ]] + log 'running db.test_remapped.find() in myApp' + set +o xtrace [2026-08-04T20:01:16+0000] running db.test_remapped.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test_remapped.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test_remapped.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rugClRdOmz +++ mktemp ++ local LAST_ERR=/tmp/tmp.ICVNhlrfPm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rugClRdOmz ++ cat /tmp/tmp.ICVNhlrfPm ++ rm /tmp/tmp.rugClRdOmz /tmp/tmp.ICVNhlrfPm ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.zea3P4E2Ii ++ mktemp + local LAST_ERR=/tmp/tmp.WjNEelKi4h + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test_remapped.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zea3P4E2Ii + cat /tmp/tmp.WjNEelKi4h + rm /tmp/tmp.zea3P4E2Ii /tmp/tmp.WjNEelKi4h + return 0 + compare_mongo_output find -remapped + local command=find + local postfix=-remapped + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-remapped.json /tmp/tmp.UFgFtUGm4s/find-remapped + return + desc 'restore from backup source, with storageName -- minio' + set +o xtrace ----------------------------------------------------------------------------------- restore from backup source, with storageName -- minio ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.mXcRYuuQFQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.QNU4wyxnAq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mXcRYuuQFQ ++ cat /tmp/tmp.QNU4wyxnAq ++ rm /tmp/tmp.mXcRYuuQFQ /tmp/tmp.QNU4wyxnAq ++ return 0 + backup_dest_minio=operator-testing/2026-08-04T19:53:05Z + run_recovery_check_bkp_source backup-minio operator-testing/2026-08-04T19:53:05Z some-name-rs0 backup-minio-source-0 + local backup=backup-minio + local backup_dest=operator-testing/2026-08-04T19:53:05Z + local cluster=some-name-rs0 + local source=backup-minio-source-0 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q2AsIvX3rT +++ mktemp ++ local LAST_ERR=/tmp/tmp.2B0B5n1TNj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Q2AsIvX3rT ++ cat /tmp/tmp.2B0B5n1TNj ++ rm /tmp/tmp.Q2AsIvX3rT /tmp/tmp.2B0B5n1TNj ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.v6Y5N0KQa6 ++ mktemp + local LAST_ERR=/tmp/tmp.CoLEFhtrsm + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v6Y5N0KQa6 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("98584204-1010-4a79-a197-3be138082f51") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.CoLEFhtrsm + rm /tmp/tmp.v6Y5N0KQa6 /tmp/tmp.CoLEFhtrsm + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:01:23+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hiAYX2ZPtx +++ mktemp ++ local LAST_ERR=/tmp/tmp.T6hUqzrqHw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hiAYX2ZPtx ++ cat /tmp/tmp.T6hUqzrqHw ++ rm /tmp/tmp.hiAYX2ZPtx /tmp/tmp.T6hUqzrqHw ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.J44qufCKQz ++ mktemp + local LAST_ERR=/tmp/tmp.J902mKhCpC + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J44qufCKQz + cat /tmp/tmp.J902mKhCpC + rm /tmp/tmp.J44qufCKQz /tmp/tmp.J902mKhCpC + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore_backupsource backup-minio-source-0 operator-testing/2026-08-04T19:53:05Z + local backupName=backup-minio-source-0 + local backupDest=operator-testing/2026-08-04T19:53:05Z + local storageName= + desc 'run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-08-04T19:53:05Z' + set +o xtrace ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-0 from backup backup-minio-source-0 destination is operator-testing/2026-08-04T19:53:05Z ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-0/' + /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-08-04T19:53:05Z|' + /usr/sbin/sed -e /storageName/d + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.F5Ug2Sj6hY ++ mktemp + local LAST_ERR=/tmp/tmp.AFy9IRyFUD + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F5Ug2Sj6hY perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-0 created + cat /tmp/tmp.AFy9IRyFUD + rm /tmp/tmp.F5Ug2Sj6hY /tmp/tmp.AFy9IRyFUD + return 0 + return + wait_restore backup-minio-source-0 some-name + local backup_name=backup-minio-source-0 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-source-0 object to be created.OK Waiting psmdb-restore/restore-backup-minio-source-0 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gW9pQM206t +++ mktemp ++ local LAST_ERR=/tmp/tmp.6Hi3GonCl0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gW9pQM206t ++ cat /tmp/tmp.6Hi3GonCl0 ++ rm /tmp/tmp.gW9pQM206t /tmp/tmp.6Hi3GonCl0 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:01:56+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pewTwvc2O9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.sItAqQhtRB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pewTwvc2O9 ++ cat /tmp/tmp.sItAqQhtRB ++ rm /tmp/tmp.pewTwvc2O9 /tmp/tmp.sItAqQhtRB ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.9UBbJe5byK ++ mktemp + local LAST_ERR=/tmp/tmp.jd4FeLgsml + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9UBbJe5byK + cat /tmp/tmp.jd4FeLgsml + rm /tmp/tmp.9UBbJe5byK /tmp/tmp.jd4FeLgsml + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:01:58+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z881y2vT8l +++ mktemp ++ local LAST_ERR=/tmp/tmp.pddxEVf5MK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z881y2vT8l ++ cat /tmp/tmp.pddxEVf5MK ++ rm /tmp/tmp.z881y2vT8l /tmp/tmp.pddxEVf5MK ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.0fcmvTtKK9 ++ mktemp + local LAST_ERR=/tmp/tmp.jOfOFStQxV + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0fcmvTtKK9 + cat /tmp/tmp.jOfOFStQxV + rm /tmp/tmp.0fcmvTtKK9 /tmp/tmp.jOfOFStQxV + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:02:01+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ty9JBQwLIL +++ mktemp ++ local LAST_ERR=/tmp/tmp.BeC530Mzr8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ty9JBQwLIL ++ cat /tmp/tmp.BeC530Mzr8 ++ rm /tmp/tmp.ty9JBQwLIL /tmp/tmp.BeC530Mzr8 ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Onah0j7OqA ++ mktemp + local LAST_ERR=/tmp/tmp.fZzkIJNdJa + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Onah0j7OqA + cat /tmp/tmp.fZzkIJNdJa + rm /tmp/tmp.Onah0j7OqA /tmp/tmp.fZzkIJNdJa + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'restore from backup source, no storageName -- minio' + set +o xtrace ----------------------------------------------------------------------------------- restore from backup source, no storageName -- minio ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oci://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.lighwRySmn +++ mktemp ++ local LAST_ERR=/tmp/tmp.vyWK87KUgz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lighwRySmn ++ cat /tmp/tmp.vyWK87KUgz ++ rm /tmp/tmp.lighwRySmn /tmp/tmp.vyWK87KUgz ++ return 0 + backup_dest_minio=operator-testing/2026-08-04T19:53:05Z + run_recovery_check_bkp_source backup-minio operator-testing/2026-08-04T19:53:05Z some-name-rs0 backup-minio-source-1 + local backup=backup-minio + local backup_dest=operator-testing/2026-08-04T19:53:05Z + local cluster=some-name-rs0 + local source=backup-minio-source-1 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.demand-backup-26283 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JOOQfneQIs +++ mktemp ++ local LAST_ERR=/tmp/tmp.DcFB14vEGS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JOOQfneQIs ++ cat /tmp/tmp.DcFB14vEGS ++ rm /tmp/tmp.JOOQfneQIs /tmp/tmp.DcFB14vEGS ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.ut9aEjm7aq ++ mktemp + local LAST_ERR=/tmp/tmp.sZoQbhoHxg + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ut9aEjm7aq Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("fd965916-1ca0-471f-94f0-e17a1241d28d") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.sZoQbhoHxg + rm /tmp/tmp.ut9aEjm7aq /tmp/tmp.sZoQbhoHxg + return 0 + compare_mongo_cmd find myApp:myPass@some-name-rs0.demand-backup-26283 -2nd .svc.cluster.local myApp test + local command=find + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:02:07+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0.demand-backup-26283 mongodb .svc.cluster.local '' '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.anBeJlN049 +++ mktemp ++ local LAST_ERR=/tmp/tmp.kJc82rzUua ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.anBeJlN049 ++ cat /tmp/tmp.kJc82rzUua ++ rm /tmp/tmp.anBeJlN049 /tmp/tmp.kJc82rzUua ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.mOZhZVf0dr ++ mktemp + local LAST_ERR=/tmp/tmp.DiuLlk5sfk + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mOZhZVf0dr + cat /tmp/tmp.DiuLlk5sfk + rm /tmp/tmp.mOZhZVf0dr /tmp/tmp.DiuLlk5sfk + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find-2nd.json /tmp/tmp.UFgFtUGm4s/find-2nd + return + run_restore_backupsource backup-minio-source-1 operator-testing/2026-08-04T19:53:05Z + local backupName=backup-minio-source-1 + local backupDest=operator-testing/2026-08-04T19:53:05Z + local storageName= + desc 'run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-08-04T19:53:05Z' + set +o xtrace ----------------------------------------------------------------------------------- run restore restore-backup-minio-source-1 from backup backup-minio-source-1 destination is operator-testing/2026-08-04T19:53:05Z ----------------------------------------------------------------------------------- + '[' -z '' ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-source-1/' + /usr/sbin/sed -e 's|BACKUP-NAME|operator-testing/2026-08-04T19:53:05Z|' + /usr/sbin/sed -e /storageName/d + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.FEyDubSZvt ++ mktemp + local LAST_ERR=/tmp/tmp.LodnuNlvM7 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FEyDubSZvt perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-source-1 created + cat /tmp/tmp.LodnuNlvM7 + rm /tmp/tmp.FEyDubSZvt /tmp/tmp.LodnuNlvM7 + return 0 + return + wait_restore backup-minio-source-1 some-name + local backup_name=backup-minio-source-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-source-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-source-1 to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SyAlYjJJ5P +++ mktemp ++ local LAST_ERR=/tmp/tmp.tWLMTqzDNv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SyAlYjJJ5P ++ cat /tmp/tmp.tWLMTqzDNv ++ rm /tmp/tmp.SyAlYjJJ5P /tmp/tmp.tWLMTqzDNv ++ return 0 + [[ ready == ready ]] + echo .OK .OK + simple_data_check some-name-rs0 3 0 + local cluster_name=some-name-rs0 + let last_pod=3-1 + local isSharded=0 + local cluster_pfx= + '[' 0 -eq 1 ']' ++ seq 0 2 + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:02:37+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.90yiGsw5jh +++ mktemp ++ local LAST_ERR=/tmp/tmp.E5PW6BbxWG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.90yiGsw5jh ++ cat /tmp/tmp.E5PW6BbxWG ++ rm /tmp/tmp.90yiGsw5jh /tmp/tmp.E5PW6BbxWG ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.9X2pB7a0cW ++ mktemp + local LAST_ERR=/tmp/tmp.5bOW42NwtL + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9X2pB7a0cW + cat /tmp/tmp.5bOW42NwtL + rm /tmp/tmp.9X2pB7a0cW /tmp/tmp.5bOW42NwtL + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:02:40+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I30iw7iWGR +++ mktemp ++ local LAST_ERR=/tmp/tmp.Oi59D8cAVj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I30iw7iWGR ++ cat /tmp/tmp.Oi59D8cAVj ++ rm /tmp/tmp.I30iw7iWGR /tmp/tmp.Oi59D8cAVj ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.J4I7UcGmrd ++ mktemp + local LAST_ERR=/tmp/tmp.8ctNjaalZQ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J4I7UcGmrd + cat /tmp/tmp.8ctNjaalZQ + rm /tmp/tmp.J4I7UcGmrd /tmp/tmp.8ctNjaalZQ + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + for i in $(seq 0 $last_pod) + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local retries=3 + local retry_delay=10 + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-08-04T20:02:43+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 mongodb '' '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6xVeqJIKVu +++ mktemp ++ local LAST_ERR=/tmp/tmp.aIxIAuwvoP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6xVeqJIKVu ++ cat /tmp/tmp.aIxIAuwvoP ++ rm /tmp/tmp.6xVeqJIKVu /tmp/tmp.aIxIAuwvoP ++ return 0 + local client_container=psmdb-client-757cd65695-k2wrh + kubectl_bin exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.C1VEZjiqR0 ++ mktemp + local LAST_ERR=/tmp/tmp.omI12DezjS + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-k2wrh' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-k2wrh -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-26283.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.C1VEZjiqR0 + cat /tmp/tmp.omI12DezjS + rm /tmp/tmp.C1VEZjiqR0 /tmp/tmp.omI12DezjS + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/compare/find.json /tmp/tmp.UFgFtUGm4s/find + return + desc 'delete backup and check if it is removed from bucket -- minio' + set +o xtrace ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.aWMtlFmI0D ++ mktemp + local LAST_ERR=/tmp/tmp.IiqdjhMYQ5 + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aWMtlFmI0D perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-26283 namespace + cat /tmp/tmp.IiqdjhMYQ5 + rm /tmp/tmp.aWMtlFmI0D /tmp/tmp.IiqdjhMYQ5 + return 0 ++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/ ++ local endpoint=http://minio-service:9000 ++ shift ++ grep -c operator-testing/2026-08-04T19:53:05Z ++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ cat +++ mktemp ++ local LAST_OUT=/tmp/tmp.6kvzUoXTCr +++ mktemp ++ local LAST_ERR=/tmp/tmp.3mSxFNjxwl ++ local exit_status=0 ++ local timeout=4 ++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6kvzUoXTCr ++ cat /tmp/tmp.3mSxFNjxwl All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-26283 ++ rm /tmp/tmp.6kvzUoXTCr /tmp/tmp.3mSxFNjxwl ++ return 0 + backup_exists=0 + [[ 0 -eq 1 ]] + '[' -z '' ']' + check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:52:54Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-08-04T19:52:54Z + key=/rs0/myApp.test.gz + storage_name=aws-s3 + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_aws_credentials + local secret_name=aws-s3-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-08-04T19:52:54Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-08-04T19:52:54Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3' Backup psmdb-demand-backup/2026-08-04T19:52:54Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3 + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:53:47Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T19:54:00Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z /rs0/myApp.test.gz ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T19:53:16Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup/2026-08-04T19:53:16Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob' Backup psmdb-demand-backup/2026-08-04T19:53:16Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob + check_backup_deletion_oss operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1 ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z + key=/rs0/myApp.test.gz ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + region=eu-central-1 + storage_name=oss-native + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native' Backup psmdb-demand-backup-oss-native/2026-08-04T19:53:29Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + desc 'checking backup deletion without cr' + set +o xtrace ----------------------------------------------------------------------------------- checking backup deletion without cr ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-08-04T20:03:06+0000] running backup backup-minio + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.8Wx7dx4xo4 ++ mktemp + local LAST_ERR=/tmp/tmp.DZncttf8Ro + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8Wx7dx4xo4 perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.DZncttf8Ro + rm /tmp/tmp.8Wx7dx4xo4 /tmp/tmp.DZncttf8Ro + return 0 + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-08-04T20:03:09+0000] running backup backup-aws-s3 + yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.RJoOVfLbAc ++ mktemp + local LAST_ERR=/tmp/tmp.b6J3Zww4um + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RJoOVfLbAc perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.b6J3Zww4um + rm /tmp/tmp.RJoOVfLbAc /tmp/tmp.b6J3Zww4um + return 0 + run_backup gcp-cs-s3 + local storage=gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local type=logical + log 'running backup backup-gcp-cs-s3' + set +o xtrace [2026-08-04T20:03:11+0000] running backup backup-gcp-cs-s3 + yq eval $'.metadata.name = "backup-gcp-cs-s3"\n\t\t\t| .spec.storageName = "gcp-cs-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-gcp-cs-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.bmSusKROL7 ++ mktemp + local LAST_ERR=/tmp/tmp.SRKSM2viFE + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bmSusKROL7 perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-s3 created + cat /tmp/tmp.SRKSM2viFE + rm /tmp/tmp.bmSusKROL7 /tmp/tmp.SRKSM2viFE + return 0 + run_backup gcp-cs-sa + local storage=gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local type=logical + log 'running backup backup-gcp-cs-sa' + set +o xtrace [2026-08-04T20:03:13+0000] running backup backup-gcp-cs-sa + yq eval $'.metadata.name = "backup-gcp-cs-sa"\n\t\t\t| .spec.storageName = "gcp-cs-sa"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-gcp-cs-sa.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.zHYwicKCLv ++ mktemp + local LAST_ERR=/tmp/tmp.f69bsvP5z3 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zHYwicKCLv perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sa created + cat /tmp/tmp.f69bsvP5z3 + rm /tmp/tmp.zHYwicKCLv /tmp/tmp.f69bsvP5z3 + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-08-04T20:03:16+0000] running backup backup-azure-blob + yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TaGS7JAnt5 ++ mktemp + local LAST_ERR=/tmp/tmp.zr2gTzRsLe + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TaGS7JAnt5 perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.zr2gTzRsLe + rm /tmp/tmp.TaGS7JAnt5 /tmp/tmp.zr2gTzRsLe + return 0 + run_backup oss-native + local storage=oss-native + local backup_name=backup-oss-native + local type=logical + log 'running backup backup-oss-native' + set +o xtrace [2026-08-04T20:03:18+0000] running backup backup-oss-native + yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.EVlTG1M8nD ++ mktemp + local LAST_ERR=/tmp/tmp.sLG7AEqJrw + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EVlTG1M8nD perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created + cat /tmp/tmp.sLG7AEqJrw + rm /tmp/tmp.EVlTG1M8nD /tmp/tmp.sLG7AEqJrw + return 0 + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state..OK + '[' -z '' ']' + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state.....................................OK + wait_backup backup-gcp-cs-s3 + local backup_name=backup-gcp-cs-s3 + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-s3 to reach ready state..OK + wait_backup backup-gcp-cs-sa + local backup_name=backup-gcp-cs-sa + local target_state=ready + set +o xtrace waiting for backup-gcp-cs-sa to reach ready state..OK + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state..OK + wait_backup backup-oss-native + local backup_name=backup-oss-native + local target_state=ready + set +o xtrace waiting for backup-oss-native to reach ready state..OK ++ get_backup_dest backup-aws-s3 ++ local backup_name=backup-aws-s3 ++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.9eBOX51R7Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.wTIc7qhLIW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9eBOX51R7Y ++ cat /tmp/tmp.wTIc7qhLIW ++ rm /tmp/tmp.9eBOX51R7Y /tmp/tmp.wTIc7qhLIW ++ return 0 + backup_dest_aws=operator-testing/psmdb-demand-backup/2026-08-04T20:04:19Z ++ get_backup_dest backup-gcp-cs-s3 ++ local backup_name=backup-gcp-cs-s3 ++ kubectl_bin get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.r0tG9Hv9ln +++ mktemp ++ local LAST_ERR=/tmp/tmp.l7zFPM5IUH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r0tG9Hv9ln ++ cat /tmp/tmp.l7zFPM5IUH ++ rm /tmp/tmp.r0tG9Hv9ln /tmp/tmp.l7zFPM5IUH ++ return 0 + backup_dest_gcp_s3=operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z ++ get_backup_dest backup-gcp-cs-sa ++ local backup_name=backup-gcp-cs-sa ++ kubectl_bin get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.yccKlyaBaA +++ mktemp ++ local LAST_ERR=/tmp/tmp.uJ5VAagtKr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs-sa' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs-sa -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yccKlyaBaA ++ cat /tmp/tmp.uJ5VAagtKr ++ rm /tmp/tmp.yccKlyaBaA /tmp/tmp.uJ5VAagtKr ++ return 0 + backup_dest_gcp_sa=operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.BU5ZYsnoPd +++ mktemp ++ local LAST_ERR=/tmp/tmp.QtmDCZCbWl ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BU5ZYsnoPd ++ cat /tmp/tmp.QtmDCZCbWl ++ rm /tmp/tmp.BU5ZYsnoPd /tmp/tmp.QtmDCZCbWl ++ return 0 + backup_dest_azure=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|oss://||' ++ sed 's|gs://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.IhgeNvjGZv +++ mktemp ++ local LAST_ERR=/tmp/tmp.CAJIMrLkra ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IhgeNvjGZv ++ cat /tmp/tmp.CAJIMrLkra ++ rm /tmp/tmp.IhgeNvjGZv /tmp/tmp.CAJIMrLkra ++ return 0 + backup_dest_oss_native=operator-testing/psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hKPbu6RHHG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.w9h2Jc3DsP +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hKPbu6RHHG +++ cat /tmp/tmp.w9h2Jc3DsP +++ rm /tmp/tmp.hKPbu6RHHG /tmp/tmp.w9h2Jc3DsP +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NPGTpl0Bw9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Pbn0hVcOfW +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.NPGTpl0Bw9 +++ cat /tmp/tmp.Pbn0hVcOfW +++ rm /tmp/tmp.NPGTpl0Bw9 /tmp/tmp.Pbn0hVcOfW +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3FZZETtMUD +++ mktemp ++ local LAST_ERR=/tmp/tmp.x8BgrDZEgL ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3FZZETtMUD ++ cat /tmp/tmp.x8BgrDZEgL ++ rm /tmp/tmp.3FZZETtMUD /tmp/tmp.x8BgrDZEgL ++ return 0 + backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NRdeWMQwbp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gFty1B9gnM +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.NRdeWMQwbp +++ cat /tmp/tmp.gFty1B9gnM +++ rm /tmp/tmp.NRdeWMQwbp /tmp/tmp.gFty1B9gnM +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yKxiTd09x1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.E2JWBNisSu +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.yKxiTd09x1 +++ cat /tmp/tmp.E2JWBNisSu +++ rm /tmp/tmp.yKxiTd09x1 /tmp/tmp.E2JWBNisSu +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lrGlyFwuON +++ mktemp ++ local LAST_ERR=/tmp/tmp.8LaexANo76 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lrGlyFwuON ++ cat /tmp/tmp.8LaexANo76 ++ rm /tmp/tmp.lrGlyFwuON /tmp/tmp.8LaexANo76 ++ return 0 + backup_region_oss_native=eu-central-1 + kubectl_bin delete psmdb --all ++ mktemp + local LAST_OUT=/tmp/tmp.YHk2xogYow ++ mktemp + local LAST_ERR=/tmp/tmp.f0nG07x82o + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YHk2xogYow perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-26283 namespace + cat /tmp/tmp.f0nG07x82o + rm /tmp/tmp.YHk2xogYow /tmp/tmp.f0nG07x82o + return 0 + sleep 60 + desc 'delete backup and check if it is removed from bucket -- minio' + set +o xtrace ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.UsnLy5duD3 ++ mktemp + local LAST_ERR=/tmp/tmp.rhihR1TBqs + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UsnLy5duD3 perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-s3" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sa" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-26283 namespace perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-26283 namespace + cat /tmp/tmp.rhihR1TBqs + rm /tmp/tmp.UsnLy5duD3 /tmp/tmp.rhihR1TBqs + return 0 ++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/ ++ local endpoint=http://minio-service:9000 ++ grep -c operator-testing/2026-08-04T19:53:05Z ++ shift ++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ cat +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ci1TWspHqA +++ mktemp ++ local LAST_ERR=/tmp/tmp.IliBqbw0LZ ++ local exit_status=0 ++ local timeout=4 ++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ci1TWspHqA ++ cat /tmp/tmp.IliBqbw0LZ All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. ++ rm /tmp/tmp.Ci1TWspHqA /tmp/tmp.IliBqbw0LZ ++ return 0 + backup_exists=0 + [[ 0 -eq 1 ]] + '[' -z '' ']' + check_backup_deletion_aws operator-testing/psmdb-demand-backup/2026-08-04T20:04:19Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup/2026-08-04T20:04:19Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T20:04:19Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup/2026-08-04T20:04:19Z + key=/rs0/myApp.test.gz + storage_name=aws-s3 + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_aws_credentials + local secret_name=aws-s3-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup/2026-08-04T20:04:19Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-08-04T20:04:19Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3' Backup psmdb-demand-backup/2026-08-04T20:04:19Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3 + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T20:03:50Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_gcs operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup/2026-08-04T20:04:03Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_azure https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z /rs0/myApp.test.gz ++ echo https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z ++ echo operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup/2026-08-04T20:03:38Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup/2026-08-04T20:03:38Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup/2026-08-04T20:03:38Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup/2026-08-04T20:03:38Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup/2026-08-04T20:03:38Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob' Backup psmdb-demand-backup/2026-08-04T20:03:38Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob + check_backup_deletion_oss operator-testing/psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1 ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z + key=/rs0/myApp.test.gz ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + region=eu-central-1 + storage_name=oss-native + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native' Backup psmdb-demand-backup-oss-native/2026-08-04T20:03:20Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + destroy demand-backup-26283 + local namespace=demand-backup-26283 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.QBo42yg9UN +++ mktemp ++ local LAST_ERR=/tmp/tmp.EyfHJNJiCR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QBo42yg9UN ++ cat /tmp/tmp.EyfHJNJiCR No resources found in demand-backup-26283 namespace. ++ rm /tmp/tmp.QBo42yg9UN /tmp/tmp.EyfHJNJiCR ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.t5qYpb1p1G ++ mktemp + local LAST_ERR=/tmp/tmp.6DTEvG817y + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t5qYpb1p1G customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.6DTEvG817y + rm /tmp/tmp.t5qYpb1p1G /tmp/tmp.6DTEvG817y + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.EaKj6ahZzP ++ mktemp + local LAST_ERR=/tmp/tmp.AWGY7Cp2Vu + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EaKj6ahZzP + cat /tmp/tmp.AWGY7Cp2Vu + rm /tmp/tmp.EaKj6ahZzP /tmp/tmp.AWGY7Cp2Vu + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Nq9wLFsHJk ++ mktemp + local LAST_ERR=/tmp/tmp.VBoquYrCx3 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Nq9wLFsHJk + cat /tmp/tmp.VBoquYrCx3 + rm /tmp/tmp.Nq9wLFsHJk /tmp/tmp.VBoquYrCx3 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.DQE8ccLnRz ++ mktemp + local LAST_ERR=/tmp/tmp.AhgtQj8V4z + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DQE8ccLnRz + cat /tmp/tmp.AhgtQj8V4z + rm /tmp/tmp.DQE8ccLnRz /tmp/tmp.AhgtQj8V4z + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.qJsBWY2nG9 ++ mktemp + local LAST_ERR=/tmp/tmp.rPCUovUzWR + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qJsBWY2nG9 + cat /tmp/tmp.rPCUovUzWR + rm /tmp/tmp.qJsBWY2nG9 /tmp/tmp.rPCUovUzWR + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.AYKQ3xPXbz ++ mktemp + local LAST_ERR=/tmp/tmp.gzVnxeEDwY + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AYKQ3xPXbz clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.gzVnxeEDwY + rm /tmp/tmp.AYKQ3xPXbz /tmp/tmp.gzVnxeEDwY + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.SJxlDzlY0C ++ mktemp + local LAST_ERR=/tmp/tmp.96V0nf4uJK + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.SJxlDzlY0C + cat /tmp/tmp.96V0nf4uJK Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.SJxlDzlY0C + cat /tmp/tmp.96V0nf4uJK Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.SJxlDzlY0C + cat /tmp/tmp.96V0nf4uJK Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.SJxlDzlY0C + cat /tmp/tmp.96V0nf4uJK Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.SJxlDzlY0C /tmp/tmp.96V0nf4uJK + return 1 + true + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-26283 + rm -rf /tmp/tmp.UFgFtUGm4s + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.l0sBcSuYvx + local LAST_OUT=/tmp/tmp.vgzNm9DUOq ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.SytXh8Vned + local LAST_ERR=/tmp/tmp.DAoxz6eRMm + local exit_status=0 + local exit_status=0 + local timeout=4 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'demand-backup-26283') + local args + local args + transformed_files=() + transformed_files=() + local transformed_files + arch_scheduling_enabled + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace demand-backup-26283 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator ==================================================================================================== [PASSED] e2e-tests/test_pytest_wrapper.py::test_bash_wrapper ==================================================================================================== --------------------------------------------- generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/reports/demand-backup.xml ---------------------------------------------- ---------------------------------------- Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/reports/demand-backup.html ---------------------------------------- ==================================================================================== 1 passed in 1458.58s (0:24:18) ==================================================================================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vgzNm9DUOq namespace "psmdb-operator" force deleted + cat /tmp/tmp.DAoxz6eRMm Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.vgzNm9DUOq /tmp/tmp.DAoxz6eRMm + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l0sBcSuYvx namespace "demand-backup-26283" force deleted + cat /tmp/tmp.SytXh8Vned Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.l0sBcSuYvx /tmp/tmp.SytXh8Vned + return 0