============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[pitr-physical-backup-source] [01:21:42.055784] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [01:21:44.547747] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + main + create_infra pitr-physical-backup-source-13946 + local ns=pitr-physical-backup-source-13946 + echo pitr-physical-backup-source-13946 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.uWUYcw6ke3 ++ mktemp + local LAST_ERR=/tmp/tmp.V6Oz8FSGR5 + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uWUYcw6ke3 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.V6Oz8FSGR5 + rm /tmp/tmp.uWUYcw6ke3 /tmp/tmp.V6Oz8FSGR5 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.gAlgjip67j ++ mktemp + local LAST_ERR=/tmp/tmp.IANCPsiezO + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gAlgjip67j + cat /tmp/tmp.IANCPsiezO + rm /tmp/tmp.gAlgjip67j /tmp/tmp.IANCPsiezO + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.GXgue9rvoV ++ mktemp + local LAST_ERR=/tmp/tmp.6coC559T32 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GXgue9rvoV + cat /tmp/tmp.6coC559T32 + rm /tmp/tmp.GXgue9rvoV /tmp/tmp.6coC559T32 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.82IOpUTOVD ++ mktemp + local LAST_ERR=/tmp/tmp.8D31zxeVqY + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.82IOpUTOVD + cat /tmp/tmp.8D31zxeVqY + rm /tmp/tmp.82IOpUTOVD /tmp/tmp.8D31zxeVqY + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.9gPG3KaGU1 ++ mktemp + local LAST_ERR=/tmp/tmp.MbK9wgyCyy + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9gPG3KaGU1 + cat /tmp/tmp.MbK9wgyCyy + rm /tmp/tmp.9gPG3KaGU1 /tmp/tmp.MbK9wgyCyy + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.HXWtoqrIn4 ++ mktemp + local LAST_ERR=/tmp/tmp.B12zJo0HQe + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HXWtoqrIn4 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.B12zJo0HQe + rm /tmp/tmp.HXWtoqrIn4 /tmp/tmp.B12zJo0HQe + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ngxw63JrqL +++ mktemp ++ local LAST_ERR=/tmp/tmp.QlymZx1OdU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ngxw63JrqL ++ cat /tmp/tmp.QlymZx1OdU Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ngxw63JrqL ++ cat /tmp/tmp.QlymZx1OdU Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Ngxw63JrqL ++ cat /tmp/tmp.QlymZx1OdU Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Ngxw63JrqL ++ cat /tmp/tmp.QlymZx1OdU Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Ngxw63JrqL /tmp/tmp.QlymZx1OdU ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + [[ -n '' ]] + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.ilsdkWYbJU + local LAST_OUT=/tmp/tmp.SNdgPDF0bf ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.WeeE40Hybf + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + local LAST_ERR=/tmp/tmp.cLl9McpGL8 + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ilsdkWYbJU + cat /tmp/tmp.cLl9McpGL8 + rm /tmp/tmp.ilsdkWYbJU /tmp/tmp.cLl9McpGL8 + return 0 namespace "demand-backup-sharded-15579" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SNdgPDF0bf namespace "psmdb-operator" deleted + cat /tmp/tmp.WeeE40Hybf + rm /tmp/tmp.SNdgPDF0bf /tmp/tmp.WeeE40Hybf + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.p4fDm5uSQ7 ++ mktemp + local LAST_ERR=/tmp/tmp.Vjx9RGla1G + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p4fDm5uSQ7 + cat /tmp/tmp.Vjx9RGla1G + rm /tmp/tmp.p4fDm5uSQ7 /tmp/tmp.Vjx9RGla1G + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.k849tCnBFx ++ mktemp + local LAST_ERR=/tmp/tmp.rvCF7YRxdm + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.k849tCnBFx namespace/psmdb-operator created + cat /tmp/tmp.rvCF7YRxdm + rm /tmp/tmp.k849tCnBFx /tmp/tmp.rvCF7YRxdm + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.OMtCLWeFX2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hJU5FHXkcQ ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OMtCLWeFX2 ++ cat /tmp/tmp.hJU5FHXkcQ ++ rm /tmp/tmp.OMtCLWeFX2 /tmp/tmp.hJU5FHXkcQ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.98Nr7sjeFW ++ mktemp + local LAST_ERR=/tmp/tmp.s9MdcZIED0 + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.98Nr7sjeFW Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9" modified. + cat /tmp/tmp.s9MdcZIED0 + rm /tmp/tmp.98Nr7sjeFW /tmp/tmp.s9MdcZIED0 + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.sPmgIaYbH3 ++ mktemp + local LAST_ERR=/tmp/tmp.9Zk2pWWcUO + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sPmgIaYbH3 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.9Zk2pWWcUO + rm /tmp/tmp.sPmgIaYbH3 /tmp/tmp.9Zk2pWWcUO + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.isVtOfE2eZ ++ mktemp + local LAST_ERR=/tmp/tmp.tDA80m8GDE + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.isVtOfE2eZ clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.tDA80m8GDE + rm /tmp/tmp.isVtOfE2eZ /tmp/tmp.tDA80m8GDE + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + add_arch_to_podspecs - + kubectl_bin apply -n psmdb-operator -f - + arch_scheduling_enabled + [[ -n '' ]] + cat - ++ mktemp + local LAST_OUT=/tmp/tmp.8uXJENzGTx ++ mktemp + local LAST_ERR=/tmp/tmp.yVSYuchAUn + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8uXJENzGTx deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.yVSYuchAUn + rm /tmp/tmp.8uXJENzGTx /tmp/tmp.yVSYuchAUn + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.r2DZuQ2bSP +++ mktemp ++ local LAST_ERR=/tmp/tmp.ct1kXe5GOK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r2DZuQ2bSP ++ cat /tmp/tmp.ct1kXe5GOK ++ rm /tmp/tmp.r2DZuQ2bSP /tmp/tmp.ct1kXe5GOK ++ return 0 + wait_operator_pod percona-server-mongodb-operator-66cd7b75cf-2h99v + local pod=percona-server-mongodb-operator-66cd7b75cf-2h99v + set +o xtrace waiting for pod/percona-server-mongodb-operator-66cd7b75cf-2h99v to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.P90dGp1PVd +++ mktemp ++ local LAST_ERR=/tmp/tmp.u3LZ6sgkFZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.P90dGp1PVd ++ cat /tmp/tmp.u3LZ6sgkFZ ++ rm /tmp/tmp.P90dGp1PVd /tmp/tmp.u3LZ6sgkFZ ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-2h99v ++ mktemp + local LAST_OUT=/tmp/tmp.ZeeeEvlZRQ ++ mktemp + local LAST_ERR=/tmp/tmp.Sd9pRJogDb + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-2h99v') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-2h99v + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZeeeEvlZRQ + cat /tmp/tmp.Sd9pRJogDb + rm /tmp/tmp.ZeeeEvlZRQ /tmp/tmp.Sd9pRJogDb + return 0 2026-07-17T01:23:10.535Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace pitr-physical-backup-source-13946 + local namespace=pitr-physical-backup-source-13946 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + [[ -n '' ]] + desc 'cleaned up old namespaces pitr-physical-backup-source-13946' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-physical-backup-source-13946 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-physical-backup-source-13946 --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.6flzZEJrEp + local LAST_OUT=/tmp/tmp.pq49ApgUmn ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.va8lT8AokB + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.3tmcJBuWqz + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'pitr-physical-backup-source-13946' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-physical-backup-source-13946 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6flzZEJrEp + cat /tmp/tmp.va8lT8AokB + rm /tmp/tmp.6flzZEJrEp /tmp/tmp.va8lT8AokB + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pq49ApgUmn + cat /tmp/tmp.3tmcJBuWqz + rm /tmp/tmp.pq49ApgUmn /tmp/tmp.3tmcJBuWqz + return 0 + kubectl_bin wait --for=delete namespace pitr-physical-backup-source-13946 ++ mktemp + local LAST_OUT=/tmp/tmp.BQZMg83oDf ++ mktemp + local LAST_ERR=/tmp/tmp.5jOQFpyikM + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'pitr-physical-backup-source-13946') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace pitr-physical-backup-source-13946 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BQZMg83oDf + cat /tmp/tmp.5jOQFpyikM + rm /tmp/tmp.BQZMg83oDf /tmp/tmp.5jOQFpyikM + return 0 + desc 'create namespace pitr-physical-backup-source-13946' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-physical-backup-source-13946 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-physical-backup-source-13946 ++ mktemp + local LAST_OUT=/tmp/tmp.6dlzVOKhXW ++ mktemp + local LAST_ERR=/tmp/tmp.xGQw8OH35H + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'pitr-physical-backup-source-13946') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-physical-backup-source-13946 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6dlzVOKhXW namespace/pitr-physical-backup-source-13946 created + cat /tmp/tmp.xGQw8OH35H + rm /tmp/tmp.6dlzVOKhXW /tmp/tmp.xGQw8OH35H + return 0 + set_kube_ctx pitr-physical-backup-source-13946 + local namespace=pitr-physical-backup-source-13946 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.o476LLPcg6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jcxSkDmuzu ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.o476LLPcg6 ++ cat /tmp/tmp.jcxSkDmuzu ++ rm /tmp/tmp.o476LLPcg6 /tmp/tmp.jcxSkDmuzu ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9 --namespace=pitr-physical-backup-source-13946 ++ mktemp + local LAST_OUT=/tmp/tmp.jeSV3eCd6Q ++ mktemp + local LAST_ERR=/tmp/tmp.hetlWogoQZ + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9' '--namespace=pitr-physical-backup-source-13946') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9 --namespace=pitr-physical-backup-source-13946 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jeSV3eCd6Q Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster9" modified. + cat /tmp/tmp.hetlWogoQZ + rm /tmp/tmp.jeSV3eCd6Q /tmp/tmp.hetlWogoQZ + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + arch_scheduling_enabled + [[ -n '' ]] + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Fri Jul 17 01:23:49 2026 NAMESPACE: pitr-physical-backup-source-13946 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-physical-backup-source-13946.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-backup-source-13946 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-backup-source-13946 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-backup-source-13946 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-backup-source-13946 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jsgp3weFBM +++ mktemp ++ local LAST_ERR=/tmp/tmp.MEsPROT6uH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=release=minio-service' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Jsgp3weFBM ++ cat /tmp/tmp.MEsPROT6uH ++ rm /tmp/tmp.Jsgp3weFBM /tmp/tmp.MEsPROT6uH ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-69gql + wait_pod minio-service-6d5f646cdc-69gql + local pod=minio-service-6d5f646cdc-69gql + set +o xtrace waiting for pod/minio-service-6d5f646cdc-69gql to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-backup-source-13946.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + aws_cli_minio http://minio-service:9000 s3 mb s3://operator-testing + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.1CwQJxkpQJ ++ mktemp + local LAST_ERR=/tmp/tmp.H1u5WAJA1M + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'mb' 's3://operator-testing') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1CwQJxkpQJ pod "aws-cli" deleted from pitr-physical-backup-source-13946 namespace + cat /tmp/tmp.H1u5WAJA1M All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.1CwQJxkpQJ /tmp/tmp.H1u5WAJA1M + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xq64TC08DL ++ mktemp + local LAST_ERR=/tmp/tmp.B8UKKRufPD + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xq64TC08DL secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.B8UKKRufPD + rm /tmp/tmp.xq64TC08DL /tmp/tmp.B8UKKRufPD + return 0 + cluster=some-name + desc 'create first PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.b34xJiGy48 + add_arch_to_psmdb - + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-13946/g + arch_scheduling_enabled + [[ -n '' ]] + cat - + local LAST_ERR=/tmp/tmp.cjpnF6nR0F + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b34xJiGy48 perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.cjpnF6nR0F + rm /tmp/tmp.b34xJiGy48 /tmp/tmp.cjpnF6nR0F + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PTfXFLrHdY +++ mktemp ++ local LAST_ERR=/tmp/tmp.3LGLsrbbjB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PTfXFLrHdY ++ cat /tmp/tmp.3LGLsrbbjB ++ rm /tmp/tmp.PTfXFLrHdY /tmp/tmp.3LGLsrbbjB ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BGwjWev8Kz +++ mktemp ++ local LAST_ERR=/tmp/tmp.c18s58uaQS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BGwjWev8Kz ++ cat /tmp/tmp.c18s58uaQS ++ rm /tmp/tmp.BGwjWev8Kz /tmp/tmp.c18s58uaQS ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5drrdsXfzV +++ mktemp ++ local LAST_ERR=/tmp/tmp.pPJvLMyuln ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5drrdsXfzV ++ cat /tmp/tmp.pPJvLMyuln ++ rm /tmp/tmp.5drrdsXfzV /tmp/tmp.pPJvLMyuln ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.......................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.M4ANoZuvNk +++ mktemp ++ local LAST_ERR=/tmp/tmp.M1Bqv8E6Si ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.M4ANoZuvNk ++ cat /tmp/tmp.M1Bqv8E6Si ++ rm /tmp/tmp.M4ANoZuvNk /tmp/tmp.M1Bqv8E6Si ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bWtpH1pzbA +++ mktemp ++ local LAST_ERR=/tmp/tmp.PCB4rDRZiz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bWtpH1pzbA ++ cat /tmp/tmp.PCB4rDRZiz ++ rm /tmp/tmp.bWtpH1pzbA /tmp/tmp.PCB4rDRZiz ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xwq0CaWilq +++ mktemp ++ local LAST_ERR=/tmp/tmp.N60YdVCsjS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xwq0CaWilq ++ cat /tmp/tmp.N60YdVCsjS ++ rm /tmp/tmp.xwq0CaWilq /tmp/tmp.N60YdVCsjS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + sleep 10 + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cezncYy5nK +++ mktemp ++ local LAST_ERR=/tmp/tmp.zyMopXheWK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cezncYy5nK ++ cat /tmp/tmp.zyMopXheWK ++ rm /tmp/tmp.cezncYy5nK /tmp/tmp.zyMopXheWK ++ return 0 + local client_container=psmdb-client-757cd65695-hx9zq + kubectl_bin exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.QpEPXFs3us ++ mktemp + local LAST_ERR=/tmp/tmp.EkdHYX8PwI + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-hx9zq' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QpEPXFs3us Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1ccd79fb-1e03-4165-8caf-0aa78e6d0aac") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.EkdHYX8PwI + rm /tmp/tmp.QpEPXFs3us /tmp/tmp.EkdHYX8PwI + return 0 + sleep 2 + write_document + local cmp_postfix= + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-07-17T01:28:15+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6SCaSvbEhf +++ mktemp ++ local LAST_ERR=/tmp/tmp.A6zCtdJCxI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6SCaSvbEhf ++ cat /tmp/tmp.A6zCtdJCxI ++ rm /tmp/tmp.6SCaSvbEhf /tmp/tmp.A6zCtdJCxI ++ return 0 + local client_container=psmdb-client-757cd65695-hx9zq + kubectl_bin exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.SHECXYIkga ++ mktemp + local LAST_ERR=/tmp/tmp.oqHafcIqIA + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-hx9zq' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SHECXYIkga Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("57a005ec-d925-4c77-b623-38991e31ccb1") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.oqHafcIqIA + rm /tmp/tmp.SHECXYIkga /tmp/tmp.oqHafcIqIA + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T01:28:17+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H4aGf2PADH +++ mktemp ++ local LAST_ERR=/tmp/tmp.P8ky9khTXQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.H4aGf2PADH ++ cat /tmp/tmp.P8ky9khTXQ ++ rm /tmp/tmp.H4aGf2PADH /tmp/tmp.P8ky9khTXQ ++ return 0 + local client_container=psmdb-client-757cd65695-hx9zq + kubectl_bin exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.JpuyVDe9iF ++ mktemp + local LAST_ERR=/tmp/tmp.V2bFGzHdKd + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-hx9zq' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-hx9zq -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JpuyVDe9iF + cat /tmp/tmp.V2bFGzHdKd + rm /tmp/tmp.JpuyVDe9iF /tmp/tmp.V2bFGzHdKd + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.2yHS0zlsUg/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-07-17T01:27:37.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-07-17T01:27:42.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-07-17T01:27:44.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2026-07-17T01:27:40.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2026-07-17T01:27:43.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2026-07-17T01:27:46.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2026-07-17T01:27:41.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2026-07-17T01:27:43.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2026-07-17T01:27:43.000+0000 I listening for the commands + echo 'Sleeping for 360 seconds' Sleeping for 360 seconds + sleep 360 + backup_name_minio=backup-minio + desc 'restore pitr type date using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr type date using backupSource ----------------------------------------------------------------------------------- + run_backup backup-minio 1 physical + local name=backup-minio + local idx=1 + local type=physical + desc 'run backup backup-minio-1' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-1 ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-1/' + /usr/sbin/sed -e 's/type:/type: physical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.b53VYahKYZ ++ mktemp + local LAST_ERR=/tmp/tmp.LjPRGMY230 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b53VYahKYZ perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created + cat /tmp/tmp.LjPRGMY230 + rm /tmp/tmp.b53VYahKYZ /tmp/tmp.LjPRGMY230 + return 0 + wait_backup backup-minio-1 + local backup_name=backup-minio-1 + local target_state=ready + set +o xtrace waiting for backup-minio-1 to reach ready state..........OK + compare_latest_restorable_time some-name-rs0 backup-minio-1 + local cluster=some-name-rs0 + local backup_name=backup-minio-1 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H9bzYNfDou ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Y5Tj29Xc14 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.H9bzYNfDou +++ cat /tmp/tmp.Y5Tj29Xc14 +++ rm /tmp/tmp.H9bzYNfDou /tmp/tmp.Y5Tj29Xc14 +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.shJ6VsOYBY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5EOdnWjdME +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.shJ6VsOYBY +++ cat /tmp/tmp.5EOdnWjdME +++ rm /tmp/tmp.shJ6VsOYBY /tmp/tmp.5EOdnWjdME +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 2 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Zw8fAayEti ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wO7tjTTIKi +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Zw8fAayEti +++ cat /tmp/tmp.wO7tjTTIKi +++ rm /tmp/tmp.Zw8fAayEti /tmp/tmp.wO7tjTTIKi +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 3 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ClbtN3Melp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2qJPQxibLG +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ClbtN3Melp +++ cat /tmp/tmp.2qJPQxibLG +++ rm /tmp/tmp.ClbtN3Melp /tmp/tmp.2qJPQxibLG +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 4 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qMG5vgnV2x ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ItSmEjlvfY +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.qMG5vgnV2x +++ cat /tmp/tmp.ItSmEjlvfY +++ rm /tmp/tmp.qMG5vgnV2x /tmp/tmp.ItSmEjlvfY +++ return 0 ++ first_timestamp=null ++ sleep 5 ++ [[ null != '' ]] ++ [[ null != null ]] ++ let retry+=1 ++ [[ 5 -gt 30 ]] ++ [[ null != '' ]] ++ [[ null != null ]] +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2jxcYBjKi9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jZfDw5rC8y +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2jxcYBjKi9 +++ cat /tmp/tmp.jZfDw5rC8y +++ rm /tmp/tmp.2jxcYBjKi9 /tmp/tmp.jZfDw5rC8y +++ return 0 ++ first_timestamp=1784252122 ++ sleep 5 ++ [[ 1784252122 != '' ]] ++ [[ 1784252122 != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8IrSo9ASYl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1nQxwBshBE +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8IrSo9ASYl +++ cat /tmp/tmp.1nQxwBshBE +++ rm /tmp/tmp.8IrSo9ASYl /tmp/tmp.1nQxwBshBE +++ return 0 ++ second_timestamp=1784252118 ++ let retry+=1 ++ [[ 6 -gt 30 ]] ++ [[ 1784252122 != '' ]] ++ [[ 1784252122 != null ]] ++ [[ 1784252122 == 1784252118 ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1Mk7TuyGlO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4N2Zvk7BXl +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1Mk7TuyGlO +++ cat /tmp/tmp.4N2Zvk7BXl +++ rm /tmp/tmp.1Mk7TuyGlO /tmp/tmp.4N2Zvk7BXl +++ return 0 ++ first_timestamp=1784252118 ++ sleep 5 ++ [[ 1784252118 != '' ]] ++ [[ 1784252118 != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8u08F8wPQG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wkwyAAwwcY +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.8u08F8wPQG +++ cat /tmp/tmp.wkwyAAwwcY +++ rm /tmp/tmp.8u08F8wPQG /tmp/tmp.wkwyAAwwcY +++ return 0 ++ second_timestamp=1784252118 ++ let retry+=1 ++ [[ 7 -gt 30 ]] ++ [[ 1784252118 != '' ]] ++ [[ 1784252118 != null ]] ++ [[ 1784252118 == 1784252118 ]] ++ /usr/sbin/date -u -d @1784252118 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-07-17T01:35:18Z ++ get_latest_restorable_time_from_backup_object backup-minio-1 ++ local backup_name=backup-minio-1 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hj4t1STPHN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4k4eGkrogY +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-minio-1' '-o' 'jsonpath={.status.latestRestorableTime}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.hj4t1STPHN +++ cat /tmp/tmp.4k4eGkrogY +++ rm /tmp/tmp.hj4t1STPHN /tmp/tmp.4k4eGkrogY +++ return 0 ++ latestRestorableTime=2026-07-17T01:35:18Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-07-17T01:35:18Z != '' ]] ++ [[ 2026-07-17T01:35:18Z != null ]] ++ echo 2026-07-17T01:35:18Z + backup_time=2026-07-17T01:35:18Z + [[ 2026-07-17T01:35:18Z != 2026\-07\-17T01\:35\:18Z ]] + reset_collection + desc 'reset data' + set +o xtrace ----------------------------------------------------------------------------------- reset data ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.remove({})' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.remove({})' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xMRQPMlu4Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.LOcy5PRiX4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xMRQPMlu4Q ++ cat /tmp/tmp.LOcy5PRiX4 ++ rm /tmp/tmp.xMRQPMlu4Q /tmp/tmp.LOcy5PRiX4 ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.lh0TOxBXz4 ++ mktemp + local LAST_ERR=/tmp/tmp.itNOzPzlvL + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.remove({})\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lh0TOxBXz4 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("3fb9347e-a4bd-4116-9364-295782144bd2") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nRemoved" : 1 }) bye + cat /tmp/tmp.itNOzPzlvL + rm /tmp/tmp.lh0TOxBXz4 /tmp/tmp.itNOzPzlvL + return 0 + sleep 2 + write_document '' 120 + local cmp_postfix= + local sleep_value=120 + log 'write initial data, read from all' + set +o xtrace [2026-07-17T01:35:53+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.59dmCv3ayZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.5CAu07GwxV ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.59dmCv3ayZ ++ cat /tmp/tmp.5CAu07GwxV ++ rm /tmp/tmp.59dmCv3ayZ /tmp/tmp.5CAu07GwxV ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.zW2yBQWgkl ++ mktemp + local LAST_ERR=/tmp/tmp.5rmbwvZR93 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zW2yBQWgkl Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("7487fd47-d2f4-49f6-8b9b-3bcc71a1c777") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.5rmbwvZR93 + rm /tmp/tmp.zW2yBQWgkl /tmp/tmp.5rmbwvZR93 + return 0 + sleep 120 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T01:37:56+0000] running db.test.command() in myApp + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mssfi40mnT +++ mktemp ++ local LAST_ERR=/tmp/tmp.i3HiZubG4I ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Mssfi40mnT ++ cat /tmp/tmp.i3HiZubG4I ++ rm /tmp/tmp.Mssfi40mnT /tmp/tmp.i3HiZubG4I ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.tv2g4hFadb ++ mktemp + local LAST_ERR=/tmp/tmp.lZkffgCWJj + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tv2g4hFadb + cat /tmp/tmp.lZkffgCWJj + rm /tmp/tmp.tv2g4hFadb /tmp/tmp.lZkffgCWJj + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.2yHS0zlsUg/find ++ run_mongos 'new Date().getTime() / 1000' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' --quiet ++ local 'command=new Date().getTime() / 1000' ++ local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag=--quiet ++ local port=27017 ++ local mongo_bin=mongo ++ grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match' ++ cut -d. -f1 +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.myNcEJzzyZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eHIIPZ6QRE +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.myNcEJzzyZ +++ cat /tmp/tmp.eHIIPZ6QRE +++ rm /tmp/tmp.myNcEJzzyZ /tmp/tmp.eHIIPZ6QRE +++ return 0 ++ local client_container=psmdb-client-757cd65695-cd992 ++ kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin --quiet' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cxLGTKjDWv +++ mktemp ++ local LAST_ERR=/tmp/tmp.NkabSh3F87 ++ local exit_status=0 ++ local timeout=4 ++ args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin --quiet') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''new Date().getTime() / 1000\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin --quiet' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cxLGTKjDWv ++ cat /tmp/tmp.NkabSh3F87 ++ rm /tmp/tmp.cxLGTKjDWv /tmp/tmp.NkabSh3F87 ++ return 0 + time_now=1784252281 + check_recovery backup-minio-1 date 1784252281 '' some-name backupSource + local backup_name=backup-minio-1 + local restore_type=date + local restore_date=1784252281 + local cmp_postfix= + local cluster_name=some-name + local backupSource=backupSource ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BAZLQlUnOo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cwfXrAqbaV +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.BAZLQlUnOo +++ cat /tmp/tmp.cwfXrAqbaV +++ rm /tmp/tmp.BAZLQlUnOo /tmp/tmp.cwfXrAqbaV +++ return 0 ++ echo 1784252236 + local latest_ts=1784252236 + desc 'write more data before restore by date' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by date ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zCwkqS6Y9F +++ mktemp ++ local LAST_ERR=/tmp/tmp.HH7rsI9LH8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zCwkqS6Y9F ++ cat /tmp/tmp.HH7rsI9LH8 ++ rm /tmp/tmp.zCwkqS6Y9F /tmp/tmp.HH7rsI9LH8 ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Yquw44bNjQ ++ mktemp + local LAST_ERR=/tmp/tmp.AyUuORQpAK + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Yquw44bNjQ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("42c69d26-456a-44c7-801f-12a5186b52a6") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.AyUuORQpAK + rm /tmp/tmp.Yquw44bNjQ /tmp/tmp.AyUuORQpAK + return 0 + [[ -n 1784252281 ]] ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + desc 'Restoring to time 2026-07-17 01:38:01' + set +o xtrace ----------------------------------------------------------------------------------- Restoring to time 2026-07-17 01:38:01 ----------------------------------------------------------------------------------- + retries=0 + [[ 1784252236 -gt 1784252281 ]] + [[ 0 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EGlMu1Ue1N ++++ mktemp +++ local LAST_ERR=/tmp/tmp.piugmp6UGu +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EGlMu1Ue1N +++ cat /tmp/tmp.piugmp6UGu +++ rm /tmp/tmp.EGlMu1Ue1N /tmp/tmp.piugmp6UGu +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=1 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 1 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bvebYNaajZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EZSWh3YIEj +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.bvebYNaajZ +++ cat /tmp/tmp.EZSWh3YIEj +++ rm /tmp/tmp.bvebYNaajZ /tmp/tmp.EZSWh3YIEj +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=2 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 2 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mSGIjS93eE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xQSm1ENBqS +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.mSGIjS93eE +++ cat /tmp/tmp.xQSm1ENBqS +++ rm /tmp/tmp.mSGIjS93eE /tmp/tmp.xQSm1ENBqS +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=3 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 3 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pFshxukL8d ++++ mktemp +++ local LAST_ERR=/tmp/tmp.60gwx0JCht +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.pFshxukL8d +++ cat /tmp/tmp.60gwx0JCht +++ rm /tmp/tmp.pFshxukL8d /tmp/tmp.60gwx0JCht +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=4 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 4 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4M1N64iVdf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bMf3O261Lp +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.4M1N64iVdf +++ cat /tmp/tmp.bMf3O261Lp +++ rm /tmp/tmp.4M1N64iVdf /tmp/tmp.bMf3O261Lp +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=5 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 5 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QhHuBOS1q0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IpGlNlHZCO +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QhHuBOS1q0 +++ cat /tmp/tmp.IpGlNlHZCO +++ rm /tmp/tmp.QhHuBOS1q0 /tmp/tmp.IpGlNlHZCO +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=6 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 6 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eSecJov7sk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qqg0Zvz1an +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.eSecJov7sk +++ cat /tmp/tmp.qqg0Zvz1an +++ rm /tmp/tmp.eSecJov7sk /tmp/tmp.qqg0Zvz1an +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=7 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 7 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.W9HvQl4K0V ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fPhmaVqcM8 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.W9HvQl4K0V +++ cat /tmp/tmp.fPhmaVqcM8 +++ rm /tmp/tmp.W9HvQl4K0V /tmp/tmp.fPhmaVqcM8 +++ return 0 ++ echo 1784252236 + latest_ts=1784252236 + retries=8 ++ format_date 1784252236 ++ local timestamp=1784252236 +++ TZ=UTC +++ /usr/sbin/date -d@1784252236 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:37:16 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:37:16) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252236 -gt 1784252281 ]] + [[ 8 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AXoSUX3sHf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qiIUmDn64E +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.AXoSUX3sHf +++ cat /tmp/tmp.qiIUmDn64E +++ rm /tmp/tmp.AXoSUX3sHf /tmp/tmp.qiIUmDn64E +++ return 0 ++ echo 1784252361 + latest_ts=1784252361 + retries=9 ++ format_date 1784252361 ++ local timestamp=1784252361 +++ TZ=UTC +++ /usr/sbin/date -d@1784252361 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:21 ++ format_date 1784252281 ++ local timestamp=1784252281 +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:38:01 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:21) to be greater than restore target (2026-07-17 01:38:01)' Waiting for last oplog chunk (2026-07-17 01:39:21) to be greater than restore target (2026-07-17 01:38:01) + sleep 10 + [[ 1784252361 -gt 1784252281 ]] + '[' -z backupSource ']' + desc 'check restore by date backupSource' + set +o xtrace ----------------------------------------------------------------------------------- check restore by date backupSource ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio-1 ++ local backup_name=backup-minio-1 ++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|azure://||' ++ sed 's|s3://||' ++ sed 's|oss://||' ++ sed 's|gs://||' ++ sed 's|oci://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wBR5pJlOYH +++ mktemp ++ local LAST_ERR=/tmp/tmp.MpaFGP2pHY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio-1' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wBR5pJlOYH ++ cat /tmp/tmp.MpaFGP2pHY ++ rm /tmp/tmp.wBR5pJlOYH /tmp/tmp.MpaFGP2pHY ++ return 0 + backup_dest=operator-testing/2026-07-17T01:34:33Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-1/' + /usr/sbin/sed -e /backupName/d + /usr/sbin/sed -e 's/pitrType:/type: date/' + '[' -z 1784252281 ']' ++ format_date 1784252281 ++ local timestamp=1784252281 + /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-07-17T01:34:33Z|' +++ TZ=UTC +++ /usr/sbin/date -d@1784252281 '+%Y-%m-%d %H:%M:%S' + kubectl_bin apply -f - + '[' -n '' ']' + yq ++ echo 2026-07-17 01:38:01 ++ mktemp ++ get_bucket_name backup-minio-1 ++ local backup_name=backup-minio-1 ++ kubectl_bin get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}' + /usr/sbin/sed -e 's/date:/date: 2026-07-17 01:38:01/' + local LAST_OUT=/tmp/tmp.lxts8UHQKl +++ mktemp ++ mktemp ++ local LAST_OUT=/tmp/tmp.fqmALcZXgL + local LAST_ERR=/tmp/tmp.bjwOz1ZZjg +++ mktemp + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ local LAST_ERR=/tmp/tmp.zfm3xa8JWq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio-1' '-o' 'jsonpath={.status.s3.bucket}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl apply -f - +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-1 -o 'jsonpath={.status.s3.bucket}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fqmALcZXgL ++ cat /tmp/tmp.zfm3xa8JWq ++ rm /tmp/tmp.fqmALcZXgL /tmp/tmp.zfm3xa8JWq ++ return 0 + /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lxts8UHQKl perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created + cat /tmp/tmp.bjwOz1ZZjg + rm /tmp/tmp.lxts8UHQKl /tmp/tmp.bjwOz1ZZjg + return 0 + wait_restore backup-minio-1 some-name requested 0 1200 + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1200 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-1 to reach state "requested" ........OK after 7 minutes + [[ 0 -eq 1 ]] + echo + wait_restore backup-minio-1 some-name ready 0 1600 + local backup_name=backup-minio-1 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=1600 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-1 object to be created.OK Waiting psmdb-restore/restore-backup-minio-1 to reach state "ready" ...OK after 2 minutes + [[ 0 -eq 1 ]] + echo + set -o xtrace + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EyiY4XSXD8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cPMKdJ39SR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EyiY4XSXD8 ++ cat /tmp/tmp.cPMKdJ39SR ++ rm /tmp/tmp.EyiY4XSXD8 /tmp/tmp.cPMKdJ39SR ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9adwj6piKQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.4izBu9TPrf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9adwj6piKQ ++ cat /tmp/tmp.4izBu9TPrf ++ rm /tmp/tmp.9adwj6piKQ /tmp/tmp.4izBu9TPrf ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H9Is3ZfD1F +++ mktemp ++ local LAST_ERR=/tmp/tmp.YZLLtG1i99 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.H9Is3ZfD1F ++ cat /tmp/tmp.YZLLtG1i99 ++ rm /tmp/tmp.H9Is3ZfD1F /tmp/tmp.YZLLtG1i99 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready............................................................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cKD5a85Yex +++ mktemp ++ local LAST_ERR=/tmp/tmp.SrYCG1s9bY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cKD5a85Yex ++ cat /tmp/tmp.SrYCG1s9bY ++ rm /tmp/tmp.cKD5a85Yex /tmp/tmp.SrYCG1s9bY ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eMMT6jvgYM +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yk3dI5skH4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eMMT6jvgYM ++ cat /tmp/tmp.Yk3dI5skH4 ++ rm /tmp/tmp.eMMT6jvgYM /tmp/tmp.Yk3dI5skH4 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aS2iBO6Rnf +++ mktemp ++ local LAST_ERR=/tmp/tmp.OzANRYbr6U ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aS2iBO6Rnf ++ cat /tmp/tmp.OzANRYbr6U ++ rm /tmp/tmp.aS2iBO6Rnf /tmp/tmp.OzANRYbr6U ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qhJ9njnEED +++ mktemp ++ local LAST_ERR=/tmp/tmp.MKh1n5Jkvs ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qhJ9njnEED ++ cat /tmp/tmp.MKh1n5Jkvs ++ rm /tmp/tmp.qhJ9njnEED /tmp/tmp.MKh1n5Jkvs ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lnKhIJxSHT +++ mktemp ++ local LAST_ERR=/tmp/tmp.QD8dK4hKER ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lnKhIJxSHT ++ cat /tmp/tmp.QD8dK4hKER ++ rm /tmp/tmp.lnKhIJxSHT /tmp/tmp.QD8dK4hKER ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UirTsPUwHs +++ mktemp ++ local LAST_ERR=/tmp/tmp.9Hqqxz7HM7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UirTsPUwHs ++ cat /tmp/tmp.9Hqqxz7HM7 ++ rm /tmp/tmp.UirTsPUwHs /tmp/tmp.9Hqqxz7HM7 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + sleep 10 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 '' + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T01:53:36+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fvkxv9kEOD +++ mktemp ++ local LAST_ERR=/tmp/tmp.mIN3PfmClQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fvkxv9kEOD ++ cat /tmp/tmp.mIN3PfmClQ ++ rm /tmp/tmp.Fvkxv9kEOD /tmp/tmp.mIN3PfmClQ ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.YNYJYWTq72 ++ mktemp + local LAST_ERR=/tmp/tmp.4pR9h1xmHn + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YNYJYWTq72 + cat /tmp/tmp.4pR9h1xmHn + rm /tmp/tmp.YNYJYWTq72 /tmp/tmp.4pR9h1xmHn + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.2yHS0zlsUg/find + desc 'delete PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- delete PSMDB cluster some-name ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb some-name ++ mktemp + local LAST_OUT=/tmp/tmp.Mqf7sYjmcI ++ mktemp + local LAST_ERR=/tmp/tmp.BccKmYRtWk + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb' 'some-name') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb some-name + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Mqf7sYjmcI perconaservermongodb.psmdb.percona.com "some-name" deleted from pitr-physical-backup-source-13946 namespace + cat /tmp/tmp.BccKmYRtWk + rm /tmp/tmp.Mqf7sYjmcI /tmp/tmp.BccKmYRtWk + return 0 + kubectl_bin delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.aytoY8mVd6 ++ mktemp + local LAST_ERR=/tmp/tmp.cEVMiKDaYk + local exit_status=0 + local timeout=4 + args=('delete' 'pvc' '-l' 'app.kubernetes.io/managed-by=percona-server-mongodb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete pvc -l app.kubernetes.io/managed-by=percona-server-mongodb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aytoY8mVd6 persistentvolumeclaim "mongod-data-some-name-cfg-0" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-cfg-1" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-cfg-2" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs0-0" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs0-1" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs0-2" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs1-0" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs1-1" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs1-2" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs2-0" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs2-1" deleted from pitr-physical-backup-source-13946 namespace persistentvolumeclaim "mongod-data-some-name-rs2-2" deleted from pitr-physical-backup-source-13946 namespace + cat /tmp/tmp.cEVMiKDaYk + rm /tmp/tmp.aytoY8mVd6 /tmp/tmp.cEVMiKDaYk + return 0 + sleep 10 + desc 'recreate PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- recreate PSMDB cluster some-name ----------------------------------------------------------------------------------- + desc 'create second PSMDB cluster some-name' + set +o xtrace ----------------------------------------------------------------------------------- create second PSMDB cluster some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/some-name-rs0.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + local LAST_OUT=/tmp/tmp.xWaWpGqvBL + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-physical-backup-source-13946/g + yq eval '.spec.upgradeOptions.apply="Never"' + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + local LAST_ERR=/tmp/tmp.JV9DDPdnLa + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + add_arch_to_psmdb - + arch_scheduling_enabled + [[ -n '' ]] + cat - + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xWaWpGqvBL perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.JV9DDPdnLa + rm /tmp/tmp.xWaWpGqvBL /tmp/tmp.JV9DDPdnLa + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready..........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L4U288U4me +++ mktemp ++ local LAST_ERR=/tmp/tmp.269ik3eDlh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L4U288U4me ++ cat /tmp/tmp.269ik3eDlh ++ rm /tmp/tmp.L4U288U4me /tmp/tmp.269ik3eDlh ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MO0r3YcRAz +++ mktemp ++ local LAST_ERR=/tmp/tmp.y0McIGszyr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MO0r3YcRAz ++ cat /tmp/tmp.y0McIGszyr ++ rm /tmp/tmp.MO0r3YcRAz /tmp/tmp.y0McIGszyr ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q5XYEkYCtY +++ mktemp ++ local LAST_ERR=/tmp/tmp.FSXWD2zAR4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q5XYEkYCtY ++ cat /tmp/tmp.FSXWD2zAR4 ++ rm /tmp/tmp.q5XYEkYCtY /tmp/tmp.FSXWD2zAR4 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready............................. + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NVwUVbddSm +++ mktemp ++ local LAST_ERR=/tmp/tmp.eajhJjj1Jo ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NVwUVbddSm ++ cat /tmp/tmp.eajhJjj1Jo ++ rm /tmp/tmp.NVwUVbddSm /tmp/tmp.eajhJjj1Jo ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UIXe1JOLiS +++ mktemp ++ local LAST_ERR=/tmp/tmp.ECpQ3q4xXD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UIXe1JOLiS ++ cat /tmp/tmp.ECpQ3q4xXD ++ rm /tmp/tmp.UIXe1JOLiS /tmp/tmp.ECpQ3q4xXD ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AfeOx79sbj +++ mktemp ++ local LAST_ERR=/tmp/tmp.hhhJotTuab ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AfeOx79sbj ++ cat /tmp/tmp.hhhJotTuab ++ rm /tmp/tmp.AfeOx79sbj /tmp/tmp.hhhJotTuab ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running some-name-rs1 3 + local name=some-name-rs1 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs1-0 + local pod=some-name-rs1-0 + set +o xtrace waiting for pod/some-name-rs1-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs1-1 + local pod=some-name-rs1-1 + set +o xtrace waiting for pod/some-name-rs1-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fmUI9QPe59 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EAsW2dpbKw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fmUI9QPe59 ++ cat /tmp/tmp.EAsW2dpbKw ++ rm /tmp/tmp.fmUI9QPe59 /tmp/tmp.EAsW2dpbKw ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs1-2 + local pod=some-name-rs1-2 + set +o xtrace waiting for pod/some-name-rs1-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6mhZwcX0Lo +++ mktemp ++ local LAST_ERR=/tmp/tmp.oQTrfzCSdT ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6mhZwcX0Lo ++ cat /tmp/tmp.oQTrfzCSdT ++ rm /tmp/tmp.6mhZwcX0Lo /tmp/tmp.oQTrfzCSdT ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GMSa2NW3oK +++ mktemp ++ local LAST_ERR=/tmp/tmp.P543tARiYi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GMSa2NW3oK ++ cat /tmp/tmp.P543tARiYi ++ rm /tmp/tmp.GMSa2NW3oK /tmp/tmp.P543tARiYi ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-rs2 3 + local name=some-name-rs2 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs2 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs2-0 + local pod=some-name-rs2-0 + set +o xtrace waiting for pod/some-name-rs2-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs2-1 + local pod=some-name-rs2-1 + set +o xtrace waiting for pod/some-name-rs2-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H2ZFQhdYE5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.nCgHeSLhYU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.H2ZFQhdYE5 ++ cat /tmp/tmp.nCgHeSLhYU ++ rm /tmp/tmp.H2ZFQhdYE5 /tmp/tmp.nCgHeSLhYU ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs2-2 + local pod=some-name-rs2-2 + set +o xtrace waiting for pod/some-name-rs2-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3SBTJqgTLJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.qusEFhQ2Aw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3SBTJqgTLJ ++ cat /tmp/tmp.qusEFhQ2Aw ++ rm /tmp/tmp.3SBTJqgTLJ /tmp/tmp.qusEFhQ2Aw ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jNwvYSYBjW +++ mktemp ++ local LAST_ERR=/tmp/tmp.idPo52tY8E ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs2")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jNwvYSYBjW ++ cat /tmp/tmp.idPo52tY8E ++ rm /tmp/tmp.jNwvYSYBjW /tmp/tmp.idPo52tY8E ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + sleep 10 + write_initial_data + desc 'create user myApp' + set +o xtrace ----------------------------------------------------------------------------------- create user myApp ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Up2ddaBrZL +++ mktemp ++ local LAST_ERR=/tmp/tmp.QmEETotATt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Up2ddaBrZL ++ cat /tmp/tmp.QmEETotATt ++ rm /tmp/tmp.Up2ddaBrZL /tmp/tmp.QmEETotATt ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.DBrCEdWPGN ++ mktemp + local LAST_ERR=/tmp/tmp.m5siYnnlGX + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DBrCEdWPGN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("c3836add-c36a-488d-a5fc-ade366eec23f") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.m5siYnnlGX + rm /tmp/tmp.DBrCEdWPGN /tmp/tmp.m5siYnnlGX + return 0 + sleep 2 + write_document + local cmp_postfix= + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-07-17T01:57:54+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CXIBAn6Gdp +++ mktemp ++ local LAST_ERR=/tmp/tmp.tAhPAVSoZ7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CXIBAn6Gdp ++ cat /tmp/tmp.tAhPAVSoZ7 ++ rm /tmp/tmp.CXIBAn6Gdp /tmp/tmp.tAhPAVSoZ7 ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.tupqhK8vTg ++ mktemp + local LAST_ERR=/tmp/tmp.MOlDpCcgLh + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tupqhK8vTg Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("a90a744c-98db-4b79-a25f-ae831281ab84") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.MOlDpCcgLh + rm /tmp/tmp.tupqhK8vTg /tmp/tmp.MOlDpCcgLh + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T01:57:57+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tc0MaYnfRs +++ mktemp ++ local LAST_ERR=/tmp/tmp.HxL5Td7VdN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tc0MaYnfRs ++ cat /tmp/tmp.HxL5Td7VdN ++ rm /tmp/tmp.tc0MaYnfRs /tmp/tmp.HxL5Td7VdN ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.U6mWqJmnsW ++ mktemp + local LAST_ERR=/tmp/tmp.S9KusZfeYv + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U6mWqJmnsW + cat /tmp/tmp.S9KusZfeYv + rm /tmp/tmp.U6mWqJmnsW /tmp/tmp.S9KusZfeYv + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find.json /tmp/tmp.2yHS0zlsUg/find + desc 'restore pitr type latest using backupSource' + set +o xtrace ----------------------------------------------------------------------------------- restore pitr type latest using backupSource ----------------------------------------------------------------------------------- + write_document -2nd + local cmp_postfix=-2nd + local sleep_value=0 + log 'write initial data, read from all' + set +o xtrace [2026-07-17T01:58:00+0000] write initial data, read from all + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tmwl0hSpVe +++ mktemp ++ local LAST_ERR=/tmp/tmp.ilYNKkA1iu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Tmwl0hSpVe ++ cat /tmp/tmp.ilYNKkA1iu ++ rm /tmp/tmp.Tmwl0hSpVe /tmp/tmp.ilYNKkA1iu ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Ig8gMz9ay1 ++ mktemp + local LAST_ERR=/tmp/tmp.PkP59tfxAL + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ig8gMz9ay1 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("e517747a-1e72-4fe0-9fed-9689c1535a78") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.PkP59tfxAL + rm /tmp/tmp.Ig8gMz9ay1 /tmp/tmp.PkP59tfxAL + return 0 + sleep 0 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 -2nd + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix=-2nd + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T01:58:02+0000] running db.test.command() in myApp + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YBG86Nu3na +++ mktemp ++ local LAST_ERR=/tmp/tmp.e9xtCi3FnF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YBG86Nu3na ++ cat /tmp/tmp.e9xtCi3FnF ++ rm /tmp/tmp.YBG86Nu3na /tmp/tmp.e9xtCi3FnF ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.NXtYUwcf8n ++ mktemp + local LAST_ERR=/tmp/tmp.wwRQYUIHas + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NXtYUwcf8n + cat /tmp/tmp.wwRQYUIHas + rm /tmp/tmp.NXtYUwcf8n /tmp/tmp.wwRQYUIHas + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find-2nd.json /tmp/tmp.2yHS0zlsUg/find-2nd + run_backup backup-minio 2 physical + local name=backup-minio + local idx=2 + local type=physical + desc 'run backup backup-minio-2' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-2 ----------------------------------------------------------------------------------- + /usr/sbin/sed -e 's/name:/name: backup-minio-2/' + /usr/sbin/sed -e 's/type:/type: physical/' + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.sSiKfssloO ++ mktemp + local LAST_ERR=/tmp/tmp.8KgElSOpsO + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sSiKfssloO perconaservermongodbbackup.psmdb.percona.com/backup-minio-2 created + cat /tmp/tmp.8KgElSOpsO + rm /tmp/tmp.sSiKfssloO /tmp/tmp.8KgElSOpsO + return 0 + wait_backup backup-minio-2 + local backup_name=backup-minio-2 + local target_state=ready + set +o xtrace waiting for backup-minio-2 to reach ready state............OK + compare_latest_restorable_time some-name-rs0 backup-minio-2 + local cluster=some-name-rs0 + local backup_name=backup-minio-2 + local latest_restorable_time + local backup_time ++ get_latest_restorable_time some-name-rs0 ++ local cluster=some-name-rs0 ++ local first_timestamp ++ local second_timestamp ++ local retry=0 ++ [[ '' != '' ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IOFBkW5aAE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NyGTS3T0gL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.IOFBkW5aAE +++ cat /tmp/tmp.NyGTS3T0gL +++ rm /tmp/tmp.IOFBkW5aAE /tmp/tmp.NyGTS3T0gL +++ return 0 ++ first_timestamp=1784252362 ++ sleep 5 ++ [[ 1784252362 != '' ]] ++ [[ 1784252362 != null ]] +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wlTtfwfoOP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CNqqzQDhE0 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wlTtfwfoOP +++ cat /tmp/tmp.CNqqzQDhE0 +++ rm /tmp/tmp.wlTtfwfoOP /tmp/tmp.CNqqzQDhE0 +++ return 0 ++ second_timestamp=1784252362 ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 1784252362 != '' ]] ++ [[ 1784252362 != null ]] ++ [[ 1784252362 == 1784252362 ]] ++ /usr/sbin/date -u -d @1784252362 +%Y-%m-%dT%H:%M:%SZ + latest_restorable_time=2026-07-17T01:39:22Z ++ get_latest_restorable_time_from_backup_object backup-minio-2 ++ local backup_name=backup-minio-2 ++ local latestRestorableTime ++ local retry=0 ++ [[ '' != '' ]] ++ sleep 5 +++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OPWWNaE2vz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.scpQ5TLp5z +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-minio-2' '-o' 'jsonpath={.status.latestRestorableTime}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.latestRestorableTime}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OPWWNaE2vz +++ cat /tmp/tmp.scpQ5TLp5z +++ rm /tmp/tmp.OPWWNaE2vz /tmp/tmp.scpQ5TLp5z +++ return 0 ++ latestRestorableTime=2026-07-17T01:39:22Z ++ let retry+=1 ++ [[ 1 -gt 30 ]] ++ [[ 2026-07-17T01:39:22Z != '' ]] ++ [[ 2026-07-17T01:39:22Z != null ]] ++ echo 2026-07-17T01:39:22Z + backup_time=2026-07-17T01:39:22Z + [[ 2026-07-17T01:39:22Z != 2026\-07\-17T01\:39\:22Z ]] + check_recovery backup-minio-2 latest '' -3rd some-name backupSource + local backup_name=backup-minio-2 + local restore_type=latest + local restore_date= + local cmp_postfix=-3rd + local cluster_name=some-name + local backupSource=backupSource ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j4ADbqrVgt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rZHn36dB9M +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.j4ADbqrVgt +++ cat /tmp/tmp.rZHn36dB9M +++ rm /tmp/tmp.j4ADbqrVgt /tmp/tmp.rZHn36dB9M +++ return 0 ++ echo 1784252362 + local latest_ts=1784252362 + desc 'write more data before restore by latest' + set +o xtrace ----------------------------------------------------------------------------------- write more data before restore by latest ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QFSxG88zIS +++ mktemp ++ local LAST_ERR=/tmp/tmp.AonVH4FO5A ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QFSxG88zIS ++ cat /tmp/tmp.AonVH4FO5A ++ rm /tmp/tmp.QFSxG88zIS /tmp/tmp.AonVH4FO5A ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.SRjHlhgWny ++ mktemp + local LAST_ERR=/tmp/tmp.ItDyKfMlhZ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SRjHlhgWny Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1abec060-4555-4fe5-b630-74cc918e6032") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ItDyKfMlhZ + rm /tmp/tmp.SRjHlhgWny /tmp/tmp.ItDyKfMlhZ + return 0 + [[ -n '' ]] + desc 'Restoring to latest' + set +o xtrace ----------------------------------------------------------------------------------- Restoring to latest ----------------------------------------------------------------------------------- ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2TxHmw40y1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JkdcGQCnZl +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.2TxHmw40y1 +++ cat /tmp/tmp.JkdcGQCnZl +++ rm /tmp/tmp.2TxHmw40y1 /tmp/tmp.JkdcGQCnZl +++ return 0 ++ echo 1784252362 + local current_ts=1784252362 + retries=0 + [[ 1784252362 -gt 1784252362 ]] + [[ 0 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lEwlUZbTIv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Iws8RCEFoV +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lEwlUZbTIv +++ cat /tmp/tmp.Iws8RCEFoV +++ rm /tmp/tmp.lEwlUZbTIv /tmp/tmp.Iws8RCEFoV +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=1 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 1 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.29DMkw7d0v ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wKfwdeIoyC +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.29DMkw7d0v +++ cat /tmp/tmp.wKfwdeIoyC +++ rm /tmp/tmp.29DMkw7d0v /tmp/tmp.wKfwdeIoyC +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=2 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 2 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UljnwTWUoH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.C6LYSfFPSt +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UljnwTWUoH +++ cat /tmp/tmp.C6LYSfFPSt +++ rm /tmp/tmp.UljnwTWUoH /tmp/tmp.C6LYSfFPSt +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=3 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 3 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UFnhJpmeIr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JMloLHWwfZ +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.UFnhJpmeIr +++ cat /tmp/tmp.JMloLHWwfZ +++ rm /tmp/tmp.UFnhJpmeIr /tmp/tmp.JMloLHWwfZ +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=4 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 4 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ME3E4COU84 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZKC3JooGtU +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ME3E4COU84 +++ cat /tmp/tmp.ZKC3JooGtU +++ rm /tmp/tmp.ME3E4COU84 /tmp/tmp.ZKC3JooGtU +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=5 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 5 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LdEJy6Q56u ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zKFDpaWPhC +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.LdEJy6Q56u +++ cat /tmp/tmp.zKFDpaWPhC +++ rm /tmp/tmp.LdEJy6Q56u /tmp/tmp.zKFDpaWPhC +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=6 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 6 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7vOIQIXahg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ipTgc0RTqL +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.7vOIQIXahg +++ cat /tmp/tmp.ipTgc0RTqL +++ rm /tmp/tmp.7vOIQIXahg /tmp/tmp.ipTgc0RTqL +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=7 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 7 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EvhjYjXrWj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dMoHPn5gMK +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.EvhjYjXrWj +++ cat /tmp/tmp.dMoHPn5gMK +++ rm /tmp/tmp.EvhjYjXrWj /tmp/tmp.dMoHPn5gMK +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=8 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 8 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9jAWQ4PkSu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5DLNgljFFD +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.9jAWQ4PkSu +++ cat /tmp/tmp.5DLNgljFFD +++ rm /tmp/tmp.9jAWQ4PkSu /tmp/tmp.5DLNgljFFD +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=9 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 9 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gqGqTi2Jsc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ii5KmQY0B0 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.gqGqTi2Jsc +++ cat /tmp/tmp.ii5KmQY0B0 +++ rm /tmp/tmp.gqGqTi2Jsc /tmp/tmp.ii5KmQY0B0 +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=10 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 10 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xU1Ge8vWiS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4e5UuENP9q +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.xU1Ge8vWiS +++ cat /tmp/tmp.4e5UuENP9q +++ rm /tmp/tmp.xU1Ge8vWiS /tmp/tmp.4e5UuENP9q +++ return 0 ++ echo 1784252362 + latest_ts=1784252362 + retries=11 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 01:39:22) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784252362 -gt 1784252362 ]] + [[ 11 -gt 30 ]] ++ get_latest_oplog_chunk_ts some-name ++ local cluster=some-name +++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5rCXD0Vebz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nLxZlXAti6 +++ local exit_status=0 +++ local timeout=4 +++ args=('exec' 'some-name-rs0-0' '-c' 'backup-agent' '--' 'pbm' 'status' '-o' 'json') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5rCXD0Vebz +++ cat /tmp/tmp.nLxZlXAti6 +++ rm /tmp/tmp.5rCXD0Vebz /tmp/tmp.nLxZlXAti6 +++ return 0 ++ echo 1784253645 + latest_ts=1784253645 + retries=12 ++ format_date 1784253645 ++ local timestamp=1784253645 +++ TZ=UTC +++ /usr/sbin/date -d@1784253645 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 02:00:45 ++ format_date 1784252362 ++ local timestamp=1784252362 +++ TZ=UTC +++ /usr/sbin/date -d@1784252362 '+%Y-%m-%d %H:%M:%S' ++ echo 2026-07-17 01:39:22 + echo 'Waiting for last oplog chunk (2026-07-17 02:00:45) to be 120 seconds older than starting chunk (2026-07-17 01:39:22)' Waiting for last oplog chunk (2026-07-17 02:00:45) to be 120 seconds older than starting chunk (2026-07-17 01:39:22) + sleep 10 + [[ 1784253645 -gt 1784252362 ]] + '[' -z backupSource ']' + desc 'check restore by latest backupSource' + set +o xtrace ----------------------------------------------------------------------------------- check restore by latest backupSource ----------------------------------------------------------------------------------- ++ get_backup_dest backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' ++ sed 's|gs://||' +++ mktemp ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.HjjxUDu9UI ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.pxh4e7ricf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio-2' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HjjxUDu9UI ++ cat /tmp/tmp.pxh4e7ricf ++ rm /tmp/tmp.HjjxUDu9UI /tmp/tmp.pxh4e7ricf ++ return 0 + backup_dest=operator-testing/2026-07-17T01:58:07Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-2/' + /usr/sbin/sed -e /backupName/d + /usr/sbin/sed -e 's/pitrType:/type: latest/' + '[' -z '' ']' + /usr/sbin/sed -e /date:/d + /usr/sbin/sed -e 's|DESTINATION|operator-testing/2026-07-17T01:58:07Z|' + kubectl_bin apply -f - + '[' -n '' ']' + yq ++ mktemp ++ get_bucket_name backup-minio-2 ++ local backup_name=backup-minio-2 ++ kubectl_bin get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}' +++ mktemp + local LAST_OUT=/tmp/tmp.3Wfy7eBnpJ ++ mktemp ++ local LAST_OUT=/tmp/tmp.WC8NbMAhSi + local LAST_ERR=/tmp/tmp.vrugbYlcSG + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] +++ mktemp ++ seq 0 2 ++ local LAST_ERR=/tmp/tmp.8i6b80xHNb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio-2' '-o' 'jsonpath={.status.s3.bucket}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl apply -f - +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-2 -o 'jsonpath={.status.s3.bucket}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WC8NbMAhSi ++ cat /tmp/tmp.8i6b80xHNb ++ rm /tmp/tmp.WC8NbMAhSi /tmp/tmp.8i6b80xHNb ++ return 0 + /usr/sbin/sed -e 's|BUCKET-NAME|operator-testing|' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3Wfy7eBnpJ perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-2 created + cat /tmp/tmp.vrugbYlcSG + rm /tmp/tmp.3Wfy7eBnpJ /tmp/tmp.vrugbYlcSG + return 0 + wait_restore backup-minio-2 some-name requested 0 1200 + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1200 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK Waiting psmdb-restore/restore-backup-minio-2 to reach state "requested" .........OK after 8 minutes + [[ 0 -eq 1 ]] + echo + wait_restore backup-minio-2 some-name ready 0 1600 + local backup_name=backup-minio-2 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=1600 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-2 object to be created.OK Waiting psmdb-restore/restore-backup-minio-2 to reach state "ready" ...OK after 2 minutes + [[ 0 -eq 1 ]] + echo + set -o xtrace + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GVxBChH5CZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.BJ1dq6QqGE ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GVxBChH5CZ ++ cat /tmp/tmp.BJ1dq6QqGE ++ rm /tmp/tmp.GVxBChH5CZ /tmp/tmp.BJ1dq6QqGE ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wHDep65WbK +++ mktemp ++ local LAST_ERR=/tmp/tmp.t1suCIk4KS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wHDep65WbK ++ cat /tmp/tmp.t1suCIk4KS ++ rm /tmp/tmp.wHDep65WbK /tmp/tmp.t1suCIk4KS ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ND4wD5ulZI +++ mktemp ++ local LAST_ERR=/tmp/tmp.K4QAJq5FtU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ND4wD5ulZI ++ cat /tmp/tmp.K4QAJq5FtU ++ rm /tmp/tmp.ND4wD5ulZI /tmp/tmp.K4QAJq5FtU ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready...................................................... + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ApHvSh9yQj +++ mktemp ++ local LAST_ERR=/tmp/tmp.gJrsA3i5TP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ApHvSh9yQj ++ cat /tmp/tmp.gJrsA3i5TP ++ rm /tmp/tmp.ApHvSh9yQj /tmp/tmp.gJrsA3i5TP ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DWzRAeHu9b +++ mktemp ++ local LAST_ERR=/tmp/tmp.wN6TfELP8W ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DWzRAeHu9b ++ cat /tmp/tmp.wN6TfELP8W ++ rm /tmp/tmp.DWzRAeHu9b /tmp/tmp.wN6TfELP8W ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DntflIN2fy +++ mktemp ++ local LAST_ERR=/tmp/tmp.TDIylks7Mg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DntflIN2fy ++ cat /tmp/tmp.TDIylks7Mg ++ rm /tmp/tmp.DntflIN2fy /tmp/tmp.TDIylks7Mg ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ejb5B1qq4Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.WTj0LfSxol ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ejb5B1qq4Q ++ cat /tmp/tmp.WTj0LfSxol ++ rm /tmp/tmp.Ejb5B1qq4Q /tmp/tmp.WTj0LfSxol ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bA9DDfU0f8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0hA7V9E2uq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bA9DDfU0f8 ++ cat /tmp/tmp.0hA7V9E2uq ++ rm /tmp/tmp.bA9DDfU0f8 /tmp/tmp.0hA7V9E2uq ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ww9e5qbVCc +++ mktemp ++ local LAST_ERR=/tmp/tmp.eL1cagwh1B ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ww9e5qbVCc ++ cat /tmp/tmp.eL1cagwh1B ++ rm /tmp/tmp.ww9e5qbVCc /tmp/tmp.eL1cagwh1B ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + sleep 10 + compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 -3rd + local command=find + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local postfix=-3rd + local suffix= + local database=myApp + local collection=test + local port=27017 + local tls=false + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-07-17T02:15:17+0000] running db.test.command() in myApp + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 mongodb '' '' 27017 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7ympGvOwJ2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.OVpSOI5Guw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7ympGvOwJ2 ++ cat /tmp/tmp.OVpSOI5Guw ++ rm /tmp/tmp.7ympGvOwJ2 /tmp/tmp.OVpSOI5Guw ++ return 0 + local client_container=psmdb-client-757cd65695-cd992 + kubectl_bin exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.q8fQ6aOdR8 ++ mktemp + local LAST_ERR=/tmp/tmp.lqJaZgNYoa + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-cd992' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-cd992 -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-backup-source-13946.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q8fQ6aOdR8 + cat /tmp/tmp.lqJaZgNYoa + rm /tmp/tmp.q8fQ6aOdR8 /tmp/tmp.lqJaZgNYoa + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-physical-backup-source/compare/find-3rd.json /tmp/tmp.2yHS0zlsUg/find-3rd + desc 'disable pitr' + set +o xtrace ----------------------------------------------------------------------------------- disable pitr ----------------------------------------------------------------------------------- + kubectl patch psmdb some-name --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": false}}}}' perconaservermongodb.psmdb.percona.com/some-name patched + sleep 20 + desc 'delete all backups' + set +o xtrace ----------------------------------------------------------------------------------- delete all backups ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.G4TIC6UE8K ++ mktemp + local LAST_ERR=/tmp/tmp.aJxFMOhanH + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.G4TIC6UE8K perconaservermongodbbackup.psmdb.percona.com "backup-minio-1" deleted from pitr-physical-backup-source-13946 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio-2" deleted from pitr-physical-backup-source-13946 namespace + cat /tmp/tmp.aJxFMOhanH + rm /tmp/tmp.G4TIC6UE8K /tmp/tmp.aJxFMOhanH + return 0 + desc 'destroy cluster' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster ----------------------------------------------------------------------------------- + destroy pitr-physical-backup-source-13946 + local namespace=pitr-physical-backup-source-13946 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.zqSFO6KUMP +++ mktemp ++ local LAST_ERR=/tmp/tmp.44ReIg3mqH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zqSFO6KUMP ++ cat /tmp/tmp.44ReIg3mqH No resources found in pitr-physical-backup-source-13946 namespace. ++ rm /tmp/tmp.zqSFO6KUMP /tmp/tmp.44ReIg3mqH ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.L2N426J5qh ++ mktemp + local LAST_ERR=/tmp/tmp.PUj5A0JzmF + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L2N426J5qh customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.PUj5A0JzmF + rm /tmp/tmp.L2N426J5qh /tmp/tmp.PUj5A0JzmF + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.UkZNejbWk1 ++ mktemp + local LAST_ERR=/tmp/tmp.9tb4p6JXKT + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UkZNejbWk1 + cat /tmp/tmp.9tb4p6JXKT + rm /tmp/tmp.UkZNejbWk1 /tmp/tmp.9tb4p6JXKT + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ivg7SMP8yu ++ mktemp + local LAST_ERR=/tmp/tmp.22Ip3FYULy + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ivg7SMP8yu + cat /tmp/tmp.22Ip3FYULy + rm /tmp/tmp.ivg7SMP8yu /tmp/tmp.22Ip3FYULy + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.TtRqsS6De1 ++ mktemp + local LAST_ERR=/tmp/tmp.MiwfycbqTn + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TtRqsS6De1 + cat /tmp/tmp.MiwfycbqTn + rm /tmp/tmp.TtRqsS6De1 /tmp/tmp.MiwfycbqTn + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.41XrZMweH8 ++ mktemp + local LAST_ERR=/tmp/tmp.VzofV866eA + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.41XrZMweH8 + cat /tmp/tmp.VzofV866eA + rm /tmp/tmp.41XrZMweH8 /tmp/tmp.VzofV866eA + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.McedjiRsP4 ++ mktemp + local LAST_ERR=/tmp/tmp.dj1FuTpE2Q + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.McedjiRsP4 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.dj1FuTpE2Q + rm /tmp/tmp.McedjiRsP4 /tmp/tmp.dj1FuTpE2Q + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Ts3S2108GW ++ mktemp + local LAST_ERR=/tmp/tmp.FRK0xrcevz + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Ts3S2108GW + cat /tmp/tmp.FRK0xrcevz Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Ts3S2108GW + cat /tmp/tmp.FRK0xrcevz Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.Ts3S2108GW + cat /tmp/tmp.FRK0xrcevz Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.Ts3S2108GW + cat /tmp/tmp.FRK0xrcevz Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.Ts3S2108GW /tmp/tmp.FRK0xrcevz + return 1 + true + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-backup-source-13946 + rm -rf /tmp/tmp.2yHS0zlsUg + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- ++ mktemp PASSED+ local LAST_OUT=/tmp/tmp.pgwBGN2cKf + local LAST_OUT=/tmp/tmp.mUsxSy9u0w ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.NVa5NpjxBt + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.VIqbWrvuTz + local exit_status=0 + local timeout=4 ++ seq 0 2 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'pitr-physical-backup-source-13946') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pitr-physical-backup-source-13946 - generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/pitr-physical-backup-source.xml - - Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/reports/pitr-physical-backup-source.html - ======================== 1 passed in 3294.65s (0:54:54) ======================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mUsxSy9u0w namespace "psmdb-operator" force deleted + cat /tmp/tmp.NVa5NpjxBt Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.mUsxSy9u0w /tmp/tmp.NVa5NpjxBt + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pgwBGN2cKf namespace "pitr-physical-backup-source-13946" force deleted + cat /tmp/tmp.VIqbWrvuTz Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.pgwBGN2cKf /tmp/tmp.VIqbWrvuTz + return 0