which: no gsed in (/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) mkdir -p e2e-tests/reports e2e-tests/logs ========================================================================================= test session starts ========================================================================================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-6.18.3-200.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'dependency': '0.6.1', 'json-report': '1.5.0', 'metadata': '3.1.1', 'html': '4.2.0'}, 'CI': 'true', 'BUILD_NUMBER': '3', 'BUILD_ID': '3', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2491/3/', 'NODE_NAME': 'hcloud-tfqjtxqnhrf6stbu', 'JOB_NAME': 'cloud-psmdb-operator/PR-2491', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2491-3', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491', 'GIT_COMMIT': '6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2491'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491 configfile: pyproject.toml plugins: dependency-0.6.1, json-report-1.5.0, metadata-3.1.1, html-4.2.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[demand-backup-sharded] [19:43:08.204178] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [19:43:09.407539] INFO Environment variables: KUBE_VERSION=1.33 PLATFORM=gke ARCH= EKS= GKE=1 AKS= OPENSHIFT= KIND= MINIKUBE= RANCHER= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a GIT_BRANCH=PR-2491 OPERATOR_VERSION=1.24.0 IMAGE=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 IMAGE_MONGOD=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 IMAGE_SEARCH=docker.io/perconalab/percona-server-mongodb-operator:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mongodb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1-1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-dev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersync-mongodb:0.9.0 IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 CERT_MANAGER_VER=1.21.0 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 +++ dirname run ++ realpath . + test_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded + . /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../functions +++ dirname /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../functions ++ source /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../vars ++++ mktemp -d +++ export tmp_dir=/tmp/tmp.hfrg7he8cN +++ tmp_dir=/tmp/tmp.hfrg7he8cN ++++ command -v gsed ++++ command -v sed +++ export sed=/usr/sbin/sed +++ sed=/usr/sbin/sed ++++ command -v gdate ++++ command -v date +++ export date=/usr/sbin/date +++ date=/usr/sbin/date ++++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded +++ export test_name=demand-backup-sharded +++ test_name=demand-backup-sharded +++ export namespace=demand-backup-sharded-12795 +++ namespace=demand-backup-sharded-12795 +++ export arbiter_namespace=demand-backup-sharded-arbiter-28347 +++ arbiter_namespace=demand-backup-sharded-arbiter-28347 +++ export replica_namespace=demand-backup-sharded-replica-2763 +++ replica_namespace=demand-backup-sharded-replica-2763 ++++ realpath /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../conf +++ export conf_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf +++ conf_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf ++++ realpath /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../.. +++ export src_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491 +++ src_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491 +++ export logs_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../logs +++ logs_dir=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../logs ++++ git rev-parse HEAD +++ export GIT_COMMIT=6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a +++ GIT_COMMIT=6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a +++ export GIT_BRANCH=PR-2491-6cfab0ed6 +++ GIT_BRANCH=PR-2491-6cfab0ed6 +++ export API=psmdb.percona.com/v1 +++ API=psmdb.percona.com/v1 ++++ awk '{print $2}' ++++++ dirname /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../vars +++++ realpath /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/../../deploy/cr.yaml ++++ grep crVersion /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cr.yaml +++ export OPERATOR_VERSION=1.24.0 +++ OPERATOR_VERSION=1.24.0 +++ export REGISTRY_NAME=docker.io +++ REGISTRY_NAME=docker.io +++ export REGISTRY_NAME_FULL=docker.io/ +++ REGISTRY_NAME_FULL=docker.io/ +++ export IMAGE=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ IMAGE=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ export IMAGE_MONGOD=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ IMAGE_MONGOD=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ export $'IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' +++ IMAGE_MONGOD_CHAIN=$'docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' +++ export IMAGE_SEARCH=docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ IMAGE_SEARCH=docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ export IMAGE_BACKUP=docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ IMAGE_BACKUP=docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ export IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1-1 +++ IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1-1 +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev-latest +++ export IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest +++ IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest +++ export IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-dev-latest +++ IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-dev-latest +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector +++ export IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE +++ local var=IMAGE +++ local val=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ [[ -z docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 ]] +++ [[ docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 == * * ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ val=docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ printf -v IMAGE %s docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_MONGOD +++ local var=IMAGE_MONGOD +++ local val=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ [[ -z docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ]] +++ [[ docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 == * * ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ val=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ printf -v IMAGE_MONGOD %s docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_MONGOD_CHAIN +++ local var=IMAGE_MONGOD_CHAIN +++ local $'val=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' +++ [[ -z docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ]] +++ [[ docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 == * * ]] +++ local out= ref +++ IFS= +++ read -r ref +++ [[ -n docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0 +++ out+=$'docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\n' +++ IFS= +++ read -r ref +++ [[ -n docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0 +++ out+=$'docker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\n' +++ IFS= +++ read -r ref +++ [[ -n docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ out+=$'docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0\n' +++ IFS= +++ read -r ref +++ val=$'docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' +++ printf -v IMAGE_MONGOD_CHAIN %s $'docker.io/perconalab/percona-server-mongodb-operator:main-mongod6.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod7.0\ndocker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0' +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_SEARCH +++ local var=IMAGE_SEARCH +++ local val=docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ [[ -z docker.io/perconalab/percona-server-mongodb-operator:main-mongot ]] +++ [[ docker.io/perconalab/percona-server-mongodb-operator:main-mongot == * * ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-mongot ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-mongot ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ val=docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ printf -v IMAGE_SEARCH %s docker.io/perconalab/percona-server-mongodb-operator:main-mongot +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_BACKUP +++ local var=IMAGE_BACKUP +++ local val=docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ [[ -z docker.io/perconalab/percona-server-mongodb-operator:main-backup ]] +++ [[ docker.io/perconalab/percona-server-mongodb-operator:main-backup == * * ]] ++++ with_registry docker.io/perconalab/percona-server-mongodb-operator:main-backup ++++ local ref=docker.io/perconalab/percona-server-mongodb-operator:main-backup ++++ case "$ref" in ++++ printf %s docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ val=docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ printf -v IMAGE_BACKUP %s docker.io/perconalab/percona-server-mongodb-operator:main-backup +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_PMM_CLIENT +++ local var=IMAGE_PMM_CLIENT +++ local val=docker.io/percona/pmm-client:2.44.1-1 +++ [[ -z docker.io/percona/pmm-client:2.44.1-1 ]] +++ [[ docker.io/percona/pmm-client:2.44.1-1 == * * ]] ++++ with_registry docker.io/percona/pmm-client:2.44.1-1 ++++ local ref=docker.io/percona/pmm-client:2.44.1-1 ++++ case "$ref" in ++++ printf %s docker.io/percona/pmm-client:2.44.1-1 +++ val=docker.io/percona/pmm-client:2.44.1-1 +++ printf -v IMAGE_PMM_CLIENT %s docker.io/percona/pmm-client:2.44.1-1 +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_PMM_SERVER +++ local var=IMAGE_PMM_SERVER +++ local val=docker.io/perconalab/pmm-server:dev-latest +++ [[ -z docker.io/perconalab/pmm-server:dev-latest ]] +++ [[ docker.io/perconalab/pmm-server:dev-latest == * * ]] ++++ with_registry docker.io/perconalab/pmm-server:dev-latest ++++ local ref=docker.io/perconalab/pmm-server:dev-latest ++++ case "$ref" in ++++ printf %s docker.io/perconalab/pmm-server:dev-latest +++ val=docker.io/perconalab/pmm-server:dev-latest +++ printf -v IMAGE_PMM_SERVER %s docker.io/perconalab/pmm-server:dev-latest +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_PMM3_CLIENT +++ local var=IMAGE_PMM3_CLIENT +++ local val=docker.io/perconalab/pmm-client:3-dev-latest +++ [[ -z docker.io/perconalab/pmm-client:3-dev-latest ]] +++ [[ docker.io/perconalab/pmm-client:3-dev-latest == * * ]] ++++ with_registry docker.io/perconalab/pmm-client:3-dev-latest ++++ local ref=docker.io/perconalab/pmm-client:3-dev-latest ++++ case "$ref" in ++++ printf %s docker.io/perconalab/pmm-client:3-dev-latest +++ val=docker.io/perconalab/pmm-client:3-dev-latest +++ printf -v IMAGE_PMM3_CLIENT %s docker.io/perconalab/pmm-client:3-dev-latest +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_PMM3_SERVER +++ local var=IMAGE_PMM3_SERVER +++ local val=docker.io/perconalab/pmm-server:3-dev-latest +++ [[ -z docker.io/perconalab/pmm-server:3-dev-latest ]] +++ [[ docker.io/perconalab/pmm-server:3-dev-latest == * * ]] ++++ with_registry docker.io/perconalab/pmm-server:3-dev-latest ++++ local ref=docker.io/perconalab/pmm-server:3-dev-latest ++++ case "$ref" in ++++ printf %s docker.io/perconalab/pmm-server:3-dev-latest +++ val=docker.io/perconalab/pmm-server:3-dev-latest +++ printf -v IMAGE_PMM3_SERVER %s docker.io/perconalab/pmm-server:3-dev-latest +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_LOGCOLLECTOR +++ local var=IMAGE_LOGCOLLECTOR +++ local val=docker.io/perconalab/fluentbit:main-logcollector +++ [[ -z docker.io/perconalab/fluentbit:main-logcollector ]] +++ [[ docker.io/perconalab/fluentbit:main-logcollector == * * ]] ++++ with_registry docker.io/perconalab/fluentbit:main-logcollector ++++ local ref=docker.io/perconalab/fluentbit:main-logcollector ++++ case "$ref" in ++++ printf %s docker.io/perconalab/fluentbit:main-logcollector +++ val=docker.io/perconalab/fluentbit:main-logcollector +++ printf -v IMAGE_LOGCOLLECTOR %s docker.io/perconalab/fluentbit:main-logcollector +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_CLUSTERSYNC +++ local var=IMAGE_CLUSTERSYNC +++ local val=docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ [[ -z docker.io/percona/percona-clustersync-mongodb:0.9.0 ]] +++ [[ docker.io/percona/percona-clustersync-mongodb:0.9.0 == * * ]] ++++ with_registry docker.io/percona/percona-clustersync-mongodb:0.9.0 ++++ local ref=docker.io/percona/percona-clustersync-mongodb:0.9.0 ++++ case "$ref" in ++++ printf %s docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ val=docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ printf -v IMAGE_CLUSTERSYNC %s docker.io/percona/percona-clustersync-mongodb:0.9.0 +++ for image in IMAGE IMAGE_MONGOD IMAGE_MONGOD_CHAIN IMAGE_SEARCH IMAGE_BACKUP IMAGE_PMM_CLIENT IMAGE_PMM_SERVER IMAGE_PMM3_CLIENT IMAGE_PMM3_SERVER IMAGE_LOGCOLLECTOR IMAGE_CLUSTERSYNC IMAGE_OPERATOR +++ qualify_image_var IMAGE_OPERATOR +++ local var=IMAGE_OPERATOR +++ local val= +++ [[ -z '' ]] +++ return ++++ kubectl version -o json ++++ jq -r '.serverVersion.major + "." + .serverVersion.minor' ++++ /usr/sbin/sed -r 's/[^0-9.]+//g' Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 +++ export KUBE_VERSION=1.33 +++ KUBE_VERSION=1.33 ++++ detect_k8s_env ++++ local provider=unknown ++++ kubectl api-resources ++++ grep -q openshift.io ++++ kubectl get nodes -o 'jsonpath={.items[0].metadata.labels}' ++++ grep -q eksctl.io ++++ kubectl get nodes -o 'jsonpath={.items[0].metadata.labels}' ++++ grep -q cloud.google.com/gke ++++ provider=gke ++++ echo gke +++ export PLATFORM=gke +++ PLATFORM=gke ++++ echo gke ++++ tr '[:lower:]' '[:upper:]' +++ export GKE=1 +++ GKE=1 +++ export ARCH= +++ ARCH= +++ [[ -z '' ]] ++++ sort -u ++++ kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' +++ node_arches=amd64 +++ [[ amd64 == arm64 ]] +++ export CERT_MANAGER_VER=1.21.0 +++ CERT_MANAGER_VER=1.21.0 +++ export MINIO_VER=5.4.0 +++ MINIO_VER=5.4.0 +++ export CHAOS_MESH_VER=2.7.1 +++ CHAOS_MESH_VER=2.7.1 +++ export PMM_SERVER_VER=9.9.9 +++ PMM_SERVER_VER=9.9.9 +++ export IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 +++ IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 +++ export SKIP_BACKUPS_TO_AWS_GCP_AZURE=1 +++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=1 +++ export UPDATE_COMPARE_FILES=0 +++ UPDATE_COMPARE_FILES=0 +++ export DELETE_CRD_ON_START=1 +++ DELETE_CRD_ON_START=1 +++ export SKIP_DELETE=0 +++ SKIP_DELETE=0 +++ [[ -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml ]] +++ export SKIP_BACKUPS_TO_AWS_GCP_AZURE= +++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= +++ setup_logging +++ [[ '' == true ]] +++ echo 'Debug logging disabled.' Debug logging disabled. ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ _AWS_ENV_STACK=() + set_debug + [[ 1 == 1 ]] + set -o xtrace + custom_port=27019 + create_infra demand-backup-sharded-12795 + local ns=demand-backup-sharded-12795 + echo demand-backup-sharded-12795 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.AZMEx3YYo8 ++ mktemp + local LAST_ERR=/tmp/tmp.BcaUtxWILe + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AZMEx3YYo8 + cat /tmp/tmp.BcaUtxWILe + rm /tmp/tmp.AZMEx3YYo8 /tmp/tmp.BcaUtxWILe + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ApfGNErTDu ++ mktemp + local LAST_ERR=/tmp/tmp.HZl0pcpRJm + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ApfGNErTDu + cat /tmp/tmp.HZl0pcpRJm + rm /tmp/tmp.ApfGNErTDu /tmp/tmp.HZl0pcpRJm + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.3wHUgkdVEs ++ mktemp + local LAST_ERR=/tmp/tmp.04kfBo2Tgk + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3wHUgkdVEs + cat /tmp/tmp.04kfBo2Tgk + rm /tmp/tmp.3wHUgkdVEs /tmp/tmp.04kfBo2Tgk + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.2sDgDOItXu ++ mktemp + local LAST_ERR=/tmp/tmp.bL5JbaLK7O + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2sDgDOItXu + cat /tmp/tmp.bL5JbaLK7O + rm /tmp/tmp.2sDgDOItXu /tmp/tmp.bL5JbaLK7O + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.T5FfzGGbYn ++ mktemp + local LAST_ERR=/tmp/tmp.fUmabSV7D9 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T5FfzGGbYn + cat /tmp/tmp.fUmabSV7D9 + rm /tmp/tmp.T5FfzGGbYn /tmp/tmp.fUmabSV7D9 + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.P3KDiy9LjU ++ mktemp + local LAST_ERR=/tmp/tmp.MAN067j6v4 + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.P3KDiy9LjU + cat /tmp/tmp.MAN067j6v4 + rm /tmp/tmp.P3KDiy9LjU /tmp/tmp.MAN067j6v4 + return 0 + check_crd_for_deletion PR-2491-6cfab0ed6 + local git_tag=PR-2491-6cfab0ed6 ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2491-6cfab0ed6/deploy/crd.yaml ++ /usr/sbin/sed s/---//g + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Engjhlw9lI +++ mktemp ++ local LAST_ERR=/tmp/tmp.5qiGKoDkjF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Engjhlw9lI ++ cat /tmp/tmp.5qiGKoDkjF Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Engjhlw9lI ++ cat /tmp/tmp.5qiGKoDkjF Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.Engjhlw9lI ++ cat /tmp/tmp.5qiGKoDkjF Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.Engjhlw9lI ++ cat /tmp/tmp.5qiGKoDkjF Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.Engjhlw9lI /tmp/tmp.5qiGKoDkjF ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + [[ -n '' ]] + desc 'cleaned up old namespaces psmdb-operator' + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.3WlPooyvyK + local LAST_OUT=/tmp/tmp.I4f5z4drOM ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.Xxod7vII26 + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.2UQh80M3KP + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3WlPooyvyK + cat /tmp/tmp.Xxod7vII26 + rm /tmp/tmp.3WlPooyvyK /tmp/tmp.Xxod7vII26 + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I4f5z4drOM + cat /tmp/tmp.2UQh80M3KP + rm /tmp/tmp.I4f5z4drOM /tmp/tmp.2UQh80M3KP + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.ujKhDrUseT ++ mktemp + local LAST_ERR=/tmp/tmp.rpLPkMnYk1 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ujKhDrUseT + cat /tmp/tmp.rpLPkMnYk1 + rm /tmp/tmp.ujKhDrUseT /tmp/tmp.rpLPkMnYk1 + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.1p6epGNmSF ++ mktemp + local LAST_ERR=/tmp/tmp.Sjf77G3Rq2 + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1p6epGNmSF namespace/psmdb-operator created + cat /tmp/tmp.Sjf77G3Rq2 + rm /tmp/tmp.1p6epGNmSF /tmp/tmp.Sjf77G3Rq2 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.pyegvEOuuL +++ mktemp ++ local LAST_ERR=/tmp/tmp.WuK8Vi4jod ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pyegvEOuuL ++ cat /tmp/tmp.WuK8Vi4jod ++ rm /tmp/tmp.pyegvEOuuL /tmp/tmp.WuK8Vi4jod ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.MBMl1Br5ig ++ mktemp + local LAST_ERR=/tmp/tmp.tTn65kU3qw + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MBMl1Br5ig Context "gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15" modified. + cat /tmp/tmp.tTn65kU3qw + rm /tmp/tmp.MBMl1Br5ig /tmp/tmp.tTn65kU3qw + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.fKpY3cig5Y ++ mktemp + local LAST_ERR=/tmp/tmp.BTT1SbIXW5 + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fKpY3cig5Y customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.BTT1SbIXW5 + rm /tmp/tmp.fKpY3cig5Y /tmp/tmp.BTT1SbIXW5 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.WQxFTDFwGq ++ mktemp + local LAST_ERR=/tmp/tmp.5MoPHl6sbs + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WQxFTDFwGq clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.5MoPHl6sbs + rm /tmp/tmp.WQxFTDFwGq /tmp/tmp.5MoPHl6sbs + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-operator.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.oCYzIH2CG1 ++ mktemp + local LAST_ERR=/tmp/tmp.5AW4gJtAKH + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oCYzIH2CG1 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.5AW4gJtAKH + rm /tmp/tmp.oCYzIH2CG1 /tmp/tmp.5AW4gJtAKH + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.nPDgPq4a6x +++ mktemp ++ local LAST_ERR=/tmp/tmp.QPM9Gwllig ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nPDgPq4a6x ++ cat /tmp/tmp.QPM9Gwllig ++ rm /tmp/tmp.nPDgPq4a6x /tmp/tmp.QPM9Gwllig ++ return 0 + wait_operator_pod percona-server-mongodb-operator-78964f6749-hr7m2 + local pod=percona-server-mongodb-operator-78964f6749-hr7m2 + set +o xtrace waiting for pod/percona-server-mongodb-operator-78964f6749-hr7m2 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.NS0bqjUi1l +++ mktemp ++ local LAST_ERR=/tmp/tmp.EPzu8CLjnb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NS0bqjUi1l ++ cat /tmp/tmp.EPzu8CLjnb ++ rm /tmp/tmp.NS0bqjUi1l /tmp/tmp.EPzu8CLjnb ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-78964f6749-hr7m2 ++ mktemp + local LAST_OUT=/tmp/tmp.zCBtTcWE0M ++ mktemp + local LAST_ERR=/tmp/tmp.hIotnBMz16 + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-78964f6749-hr7m2') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-78964f6749-hr7m2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zCBtTcWE0M + cat /tmp/tmp.hIotnBMz16 + rm /tmp/tmp.zCBtTcWE0M /tmp/tmp.hIotnBMz16 + return 0 2026-08-04T19:44:14.481Z INFO setup Manager starting up {"gitCommit": "6cfab0ed6bdcd3601b0c48816a22ee4ab446b92a", "gitBranch": "PR-2491-6cfab0ed6", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-sharded-12795 + local namespace=demand-backup-sharded-12795 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.v5t0i68iG2 + [[ -n '' ]] + desc 'cleaned up old namespaces demand-backup-sharded-12795' ++ mktemp + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-sharded-12795 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-sharded-12795 --ignore-not-found + awk '{print$1}' + local LAST_ERR=/tmp/tmp.F0OsVhpK6u + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + xargs kubectl delete ns + for i in $(seq 0 2) + set +e + kubectl get ns ++ mktemp + local LAST_OUT=/tmp/tmp.8Or6GsJse5 ++ mktemp + local LAST_ERR=/tmp/tmp.00HB7UhPpY + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'demand-backup-sharded-12795' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-sharded-12795 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.v5t0i68iG2 + cat /tmp/tmp.F0OsVhpK6u + rm /tmp/tmp.v5t0i68iG2 /tmp/tmp.F0OsVhpK6u + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8Or6GsJse5 + cat /tmp/tmp.00HB7UhPpY + rm /tmp/tmp.8Or6GsJse5 /tmp/tmp.00HB7UhPpY + return 0 + kubectl_bin wait --for=delete namespace demand-backup-sharded-12795 ++ mktemp + local LAST_OUT=/tmp/tmp.dp80dcPlbZ ++ mktemp + local LAST_ERR=/tmp/tmp.KLSR1MFE0R + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'demand-backup-sharded-12795') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-sharded-12795 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dp80dcPlbZ + cat /tmp/tmp.KLSR1MFE0R + rm /tmp/tmp.dp80dcPlbZ /tmp/tmp.KLSR1MFE0R + return 0 + desc 'create namespace demand-backup-sharded-12795' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-sharded-12795 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-sharded-12795 ++ mktemp + local LAST_OUT=/tmp/tmp.wzzLuCmN40 ++ mktemp + local LAST_ERR=/tmp/tmp.32rCCchGGp + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'demand-backup-sharded-12795') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-sharded-12795 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wzzLuCmN40 namespace/demand-backup-sharded-12795 created + cat /tmp/tmp.32rCCchGGp + rm /tmp/tmp.wzzLuCmN40 /tmp/tmp.32rCCchGGp + return 0 + set_kube_ctx demand-backup-sharded-12795 + local namespace=demand-backup-sharded-12795 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.H2O9rKVN2Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.a319D9tsgd ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.H2O9rKVN2Q ++ cat /tmp/tmp.a319D9tsgd ++ rm /tmp/tmp.H2O9rKVN2Q /tmp/tmp.a319D9tsgd ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15 --namespace=demand-backup-sharded-12795 ++ mktemp + local LAST_OUT=/tmp/tmp.RY3eK0RyxP ++ mktemp + local LAST_ERR=/tmp/tmp.vK98p7m5Yj + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15' '--namespace=demand-backup-sharded-12795') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15 --namespace=demand-backup-sharded-12795 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RY3eK0RyxP Context "gke_cloud-dev-112233_us-central1-c_jen-psmdb-2491-6cfab0ed6-3-cluster15" modified. + cat /tmp/tmp.vK98p7m5Yj + rm /tmp/tmp.RY3eK0RyxP /tmp/tmp.vK98p7m5Yj + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + arch_scheduling_enabled + [[ -n '' ]] + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Tue Aug 4 19:44:49 2026 NAMESPACE: demand-backup-sharded-12795 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-sharded-12795.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-12795 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-12795 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-12795 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-12795 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1G1V1PzKRD +++ mktemp ++ local LAST_ERR=/tmp/tmp.MtnxSVemW2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=release=minio-service' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1G1V1PzKRD ++ cat /tmp/tmp.MtnxSVemW2 ++ rm /tmp/tmp.1G1V1PzKRD /tmp/tmp.MtnxSVemW2 ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-4wp6z + wait_pod minio-service-6d5f646cdc-4wp6z + local pod=minio-service-6d5f646cdc-4wp6z + set +o xtrace waiting for pod/minio-service-6d5f646cdc-4wp6z to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-12795.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + aws_cli_minio http://minio-service:9000 s3 mb s3://operator-testing + local endpoint=http://minio-service:9000 + shift + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing ++ mktemp + local LAST_OUT=/tmp/tmp.e3hqGtblZh ++ mktemp + local LAST_ERR=/tmp/tmp.WodPL3A8ru + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'mb' 's3://operator-testing') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.e3hqGtblZh make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-sharded-12795 namespace + cat /tmp/tmp.WodPL3A8ru All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.e3hqGtblZh /tmp/tmp.WodPL3A8ru + return 0 + desc 'create first PSMDB cluster with custom port 27019' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster with custom port 27019 ----------------------------------------------------------------------------------- + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rqspo7s4IK ++ mktemp + local LAST_ERR=/tmp/tmp.0sFedieGIF + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rqspo7s4IK secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.0sFedieGIF + rm /tmp/tmp.rqspo7s4IK /tmp/tmp.0sFedieGIF + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.HbrvIJdnHN ++ mktemp + local LAST_ERR=/tmp/tmp.AOedsrtJu4 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HbrvIJdnHN secret/minio-secret created secret/aws-s3-secret created secret/alibaba-cloud-secret created secret/oci-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.AOedsrtJu4 + rm /tmp/tmp.HbrvIJdnHN /tmp/tmp.AOedsrtJu4 + return 0 + create_runtime_class + kubectl_bin delete runtimeclass container-rc --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.U3VMPQcSZM ++ mktemp + local LAST_ERR=/tmp/tmp.oej84VAO80 + local exit_status=0 + local timeout=4 + args=('delete' 'runtimeclass' 'container-rc' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete runtimeclass container-rc --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U3VMPQcSZM + cat /tmp/tmp.oej84VAO80 + rm /tmp/tmp.U3VMPQcSZM /tmp/tmp.oej84VAO80 + return 0 + local handler=runc + desc 'Create RuntimeClass with handler runc' + set +o xtrace ----------------------------------------------------------------------------------- Create RuntimeClass with handler runc ----------------------------------------------------------------------------------- + yq eval '.handler = "runc"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/conf/container-rc.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.wV50ErcxZe ++ mktemp + local LAST_ERR=/tmp/tmp.Yi6x9d1w5G + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wV50ErcxZe runtimeclass.node.k8s.io/container-rc created + cat /tmp/tmp.Yi6x9d1w5G + rm /tmp/tmp.wV50ErcxZe /tmp/tmp.Yi6x9d1w5G + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + kubectl_bin apply -f - + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2491-6cfab0ed6"' + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + yq eval '(.spec | select(has("search"))).search.image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongot"' + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + local LAST_OUT=/tmp/tmp.5SKwiS0y8O + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-sharded-12795/g + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + add_arch_to_psmdb - + arch_scheduling_enabled + [[ -n '' ]] + cat - + local LAST_ERR=/tmp/tmp.R9CfnivTUw + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5SKwiS0y8O perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.R9CfnivTUw + rm /tmp/tmp.5SKwiS0y8O /tmp/tmp.R9CfnivTUw + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.....................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RdpKBrmyWt +++ mktemp ++ local LAST_ERR=/tmp/tmp.BUn9TGc7St ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RdpKBrmyWt ++ cat /tmp/tmp.BUn9TGc7St ++ rm /tmp/tmp.RdpKBrmyWt /tmp/tmp.BUn9TGc7St ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FozzRIqkhj +++ mktemp ++ local LAST_ERR=/tmp/tmp.cPuN5gmXwZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FozzRIqkhj ++ cat /tmp/tmp.cPuN5gmXwZ ++ rm /tmp/tmp.FozzRIqkhj /tmp/tmp.cPuN5gmXwZ ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.clDLfeiLHF +++ mktemp ++ local LAST_ERR=/tmp/tmp.LVdZOak4wV ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.clDLfeiLHF ++ cat /tmp/tmp.LVdZOak4wV ++ rm /tmp/tmp.clDLfeiLHF /tmp/tmp.LVdZOak4wV ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cwMO3Mk12c +++ mktemp ++ local LAST_ERR=/tmp/tmp.tws06r9B7W ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cwMO3Mk12c ++ cat /tmp/tmp.tws06r9B7W ++ rm /tmp/tmp.cwMO3Mk12c /tmp/tmp.tws06r9B7W ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ssdf2gJrto +++ mktemp ++ local LAST_ERR=/tmp/tmp.jiosJfrUmM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ssdf2gJrto ++ cat /tmp/tmp.jiosJfrUmM ++ rm /tmp/tmp.ssdf2gJrto /tmp/tmp.jiosJfrUmM ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fzMaexRQ7l +++ mktemp ++ local LAST_ERR=/tmp/tmp.r9ufE10giE ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fzMaexRQ7l ++ cat /tmp/tmp.r9ufE10giE ++ rm /tmp/tmp.fzMaexRQ7l /tmp/tmp.r9ufE10giE ++ return 0 + [[ '' == true ]] + sleep 10 + [[ false == true ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tDwSxK7X1K +++ mktemp ++ local LAST_ERR=/tmp/tmp.SiSbnwFjO9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tDwSxK7X1K ++ cat /tmp/tmp.SiSbnwFjO9 ++ rm /tmp/tmp.tDwSxK7X1K /tmp/tmp.SiSbnwFjO9 ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GZu8y81ZAF +++ mktemp ++ local LAST_ERR=/tmp/tmp.pVkQoRzdJH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GZu8y81ZAF ++ cat /tmp/tmp.pVkQoRzdJH ++ rm /tmp/tmp.GZu8y81ZAF /tmp/tmp.pVkQoRzdJH ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tkI8J6rvr9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LcxvnPOOdk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tkI8J6rvr9 ++ cat /tmp/tmp.LcxvnPOOdk ++ rm /tmp/tmp.tkI8J6rvr9 /tmp/tmp.LcxvnPOOdk ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + sleep 10 + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 ++ mktemp + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.BfYEoGLwBs ++ mktemp + local LAST_ERR=/tmp/tmp.4O8bxPQZye + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BfYEoGLwBs + cat /tmp/tmp.4O8bxPQZye + rm /tmp/tmp.BfYEoGLwBs /tmp/tmp.4O8bxPQZye + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-08-04T19:49:50+0000] compare_kubectl: statefulset/some-name-rs0 OK + compare_kubectl statefulset/some-name-rs1 + local resource=statefulset/some-name-rs1 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-rs1.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs1 ++ mktemp + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.GnHfbC9bMu ++ mktemp + local LAST_ERR=/tmp/tmp.FeAi41lQ5J + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs1') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GnHfbC9bMu + cat /tmp/tmp.FeAi41lQ5J + rm /tmp/tmp.GnHfbC9bMu /tmp/tmp.FeAi41lQ5J + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs1.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs1.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs1.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs1.yml + log 'compare_kubectl: statefulset/some-name-rs1 OK' + set +o xtrace [2026-08-04T19:49:51+0000] compare_kubectl: statefulset/some-name-rs1 OK + compare_kubectl statefulset/some-name-rs2 + local resource=statefulset/some-name-rs2 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-rs2.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs2 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.sZutsjfl6k ++ mktemp + local LAST_ERR=/tmp/tmp.XI890rcFrx + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs2') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sZutsjfl6k + cat /tmp/tmp.XI890rcFrx + rm /tmp/tmp.sZutsjfl6k /tmp/tmp.XI890rcFrx + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs2.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs2.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs2.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-rs2.yml + log 'compare_kubectl: statefulset/some-name-rs2 OK' + set +o xtrace [2026-08-04T19:49:52+0000] compare_kubectl: statefulset/some-name-rs2 OK + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-cfg.yml + [[ -n '' ]] + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + kubectl_bin get -o yaml statefulset/some-name-cfg ++ mktemp + local LAST_OUT=/tmp/tmp.HBlnvT9A46 ++ mktemp + local LAST_ERR=/tmp/tmp.Mfoskg9vVE + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HBlnvT9A46 + cat /tmp/tmp.Mfoskg9vVE + rm /tmp/tmp.HBlnvT9A46 /tmp/tmp.Mfoskg9vVE + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-cfg.yml + log 'compare_kubectl: statefulset/some-name-cfg OK' + set +o xtrace [2026-08-04T19:49:53+0000] compare_kubectl: statefulset/some-name-cfg OK + compare_kubectl statefulset/some-name-mongos '' + local resource=statefulset/some-name-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.nEwUFa01bz ++ mktemp + local LAST_ERR=/tmp/tmp.ZNEcgoSsB6 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nEwUFa01bz + cat /tmp/tmp.ZNEcgoSsB6 + rm /tmp/tmp.nEwUFa01bz /tmp/tmp.ZNEcgoSsB6 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + log 'compare_kubectl: statefulset/some-name-mongos OK' + set +o xtrace [2026-08-04T19:49:54+0000] compare_kubectl: statefulset/some-name-mongos OK + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.T0uECafAfB ++ mktemp + local LAST_ERR=/tmp/tmp.qaxciMVUxW + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T0uECafAfB secret/some-name-mongos created + cat /tmp/tmp.qaxciMVUxW + rm /tmp/tmp.T0uECafAfB /tmp/tmp.qaxciMVUxW + return 0 + sleep 10 + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qcr2uDMeRK +++ mktemp ++ local LAST_ERR=/tmp/tmp.tdwJqUZUNj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Qcr2uDMeRK ++ cat /tmp/tmp.tdwJqUZUNj ++ rm /tmp/tmp.Qcr2uDMeRK /tmp/tmp.tdwJqUZUNj ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VPKwdgZE7A +++ mktemp ++ local LAST_ERR=/tmp/tmp.gsHpUa9fNh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VPKwdgZE7A ++ cat /tmp/tmp.gsHpUa9fNh ++ rm /tmp/tmp.VPKwdgZE7A /tmp/tmp.gsHpUa9fNh ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XsUfm0ZDxj +++ mktemp ++ local LAST_ERR=/tmp/tmp.WYxCZ0Mg1H ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XsUfm0ZDxj ++ cat /tmp/tmp.WYxCZ0Mg1H ++ rm /tmp/tmp.XsUfm0ZDxj /tmp/tmp.WYxCZ0Mg1H ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.................................................................................................. + compare_kubectl statefulset/some-name-mongos -secret + local resource=statefulset/some-name-mongos + local postfix=-secret + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml + local new_result=/tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-mongos + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-12795", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.ynKaw9ETp3 ++ mktemp + local LAST_ERR=/tmp/tmp.xfGsFbKPZq + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-mongos') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ynKaw9ETp3 + cat /tmp/tmp.xfGsFbKPZq + rm /tmp/tmp.ynKaw9ETp3 /tmp/tmp.xfGsFbKPZq + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.hfrg7he8cN/statefulset_some-name-mongos.yml + log 'compare_kubectl: statefulset/some-name-mongos OK' + set +o xtrace [2026-08-04T19:53:54+0000] compare_kubectl: statefulset/some-name-mongos OK + desc 'checking hookscripts' + set +o xtrace ----------------------------------------------------------------------------------- checking hookscripts ----------------------------------------------------------------------------------- + kubectl_bin logs some-name-rs0-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.d2BbRluhlC ++ mktemp + local LAST_ERR=/tmp/tmp.G3Q8xJz1ZX + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs0-0' 'backup-agent') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs0-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.d2BbRluhlC + cat /tmp/tmp.G3Q8xJz1ZX + rm /tmp/tmp.d2BbRluhlC /tmp/tmp.G3Q8xJz1ZX + return 0 + kubectl_bin logs some-name-rs0-0 + grep -q 'RS0 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.m3toGwFxoG ++ mktemp + local LAST_ERR=/tmp/tmp.CPRc0MF862 + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs0-0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs0-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.m3toGwFxoG + cat /tmp/tmp.CPRc0MF862 Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) + rm /tmp/tmp.m3toGwFxoG /tmp/tmp.CPRc0MF862 + return 0 + kubectl_bin logs some-name-rs1-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.WZapReIzZU ++ mktemp + local LAST_ERR=/tmp/tmp.gQT8YKVjHQ + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs1-0' 'backup-agent') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs1-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WZapReIzZU + cat /tmp/tmp.gQT8YKVjHQ + rm /tmp/tmp.WZapReIzZU /tmp/tmp.gQT8YKVjHQ + return 0 + kubectl_bin logs some-name-rs1-0 + grep -q 'RS1 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.TcvMTHDWOa ++ mktemp + local LAST_ERR=/tmp/tmp.l4Xs0o6Aqv + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs1-0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs1-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TcvMTHDWOa + cat /tmp/tmp.l4Xs0o6Aqv Defaulted container "mongod" out of: mongod, backup-agent, mongo-init (init) + rm /tmp/tmp.TcvMTHDWOa /tmp/tmp.l4Xs0o6Aqv + return 0 + kubectl_bin logs some-name-rs2-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.bZg2FEdJRR ++ mktemp + local LAST_ERR=/tmp/tmp.5nyEPWZ3KZ + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs2-0' 'backup-agent') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs2-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bZg2FEdJRR + cat /tmp/tmp.5nyEPWZ3KZ + rm /tmp/tmp.bZg2FEdJRR /tmp/tmp.5nyEPWZ3KZ + return 0 + kubectl_bin logs some-name-rs2-0 + grep -q 'RS2 Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.j6543TsVDh ++ mktemp + local LAST_ERR=/tmp/tmp.eX3tRLa2EE + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-rs2-0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-rs2-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j6543TsVDh + cat /tmp/tmp.eX3tRLa2EE Defaulted container "mongod" out of: mongod, rs-sidecar-1, backup-agent, mongo-init (init) + rm /tmp/tmp.j6543TsVDh /tmp/tmp.eX3tRLa2EE + return 0 + kubectl_bin logs some-name-cfg-0 backup-agent + grep -q 'Backup Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.isFFphS4zi ++ mktemp + local LAST_ERR=/tmp/tmp.CL7SUwaWPU + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-cfg-0' 'backup-agent') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-cfg-0 backup-agent + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.isFFphS4zi + cat /tmp/tmp.CL7SUwaWPU + rm /tmp/tmp.isFFphS4zi /tmp/tmp.CL7SUwaWPU + return 0 + kubectl_bin logs some-name-cfg-0 + grep -q 'CFG Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.GDQT8AvlNO ++ mktemp + local LAST_ERR=/tmp/tmp.sTbMnqS0IO + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-cfg-0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-cfg-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GDQT8AvlNO + cat /tmp/tmp.sTbMnqS0IO Defaulted container "mongod" out of: mongod, cfg-sidecar-1, backup-agent, mongo-init (init) + rm /tmp/tmp.GDQT8AvlNO /tmp/tmp.sTbMnqS0IO + return 0 + kubectl_bin logs some-name-mongos-0 + grep -q 'Mongos Hookscript started' ++ mktemp + local LAST_OUT=/tmp/tmp.Wa8AtJVBOc ++ mktemp + local LAST_ERR=/tmp/tmp.btYEW95lT2 + local exit_status=0 + local timeout=4 + args=('logs' 'some-name-mongos-0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs some-name-mongos-0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Wa8AtJVBOc + cat /tmp/tmp.btYEW95lT2 Defaulted container "mongos" out of: mongos, mongos-sidecar-1, mongo-init (init) + rm /tmp/tmp.Wa8AtJVBOc /tmp/tmp.btYEW95lT2 + return 0 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1axCHVqrzH +++ mktemp ++ local LAST_ERR=/tmp/tmp.p1NV0nzXRW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1axCHVqrzH ++ cat /tmp/tmp.p1NV0nzXRW ++ rm /tmp/tmp.1axCHVqrzH /tmp/tmp.p1NV0nzXRW ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.s8N23eMTRr ++ mktemp + local LAST_ERR=/tmp/tmp.BLf3P3Ao0x + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s8N23eMTRr Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("0c991398-58c6-43b5-b82d-825fbbd05a06") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" }, { "db" : "myApp1", "role" : "readWrite" }, { "db" : "myApp2", "role" : "readWrite" } ] } bye + cat /tmp/tmp.BLf3P3Ao0x + rm /tmp/tmp.s8N23eMTRr /tmp/tmp.BLf3P3Ao0x + return 0 + run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=sh.enableSharding("myApp","rs0")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OAPp2tj51H +++ mktemp ++ local LAST_ERR=/tmp/tmp.I6rlMhfi3l ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OAPp2tj51H ++ cat /tmp/tmp.I6rlMhfi3l ++ rm /tmp/tmp.OAPp2tj51H /tmp/tmp.I6rlMhfi3l ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.nebdVMsZ7Y ++ mktemp + local LAST_ERR=/tmp/tmp.gNdIfgJpL8 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nebdVMsZ7Y Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ecb79781-5cbf-444e-a502-d7e1d92a2711") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1785873254, 9), "signature" : { "hash" : BinData(0,"rW1um76axZdbObzV5oxy4WdqE4k="), "keyId" : NumberLong("7670265726082482200") } }, "operationTime" : Timestamp(1785873254, 6) } bye + cat /tmp/tmp.gNdIfgJpL8 + rm /tmp/tmp.nebdVMsZ7Y /tmp/tmp.gNdIfgJpL8 + return 0 + run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=sh.enableSharding("myApp1","rs1")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SoIl7cXBSD +++ mktemp ++ local LAST_ERR=/tmp/tmp.fYPC2khqMK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SoIl7cXBSD ++ cat /tmp/tmp.fYPC2khqMK ++ rm /tmp/tmp.SoIl7cXBSD /tmp/tmp.fYPC2khqMK ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.wFHmcDn2Z7 ++ mktemp + local LAST_ERR=/tmp/tmp.GvWx7cRn9T + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wFHmcDn2Z7 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("c394fe8d-7ebc-42bd-8631-e33bf6ccb3ed") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1785873256, 12), "signature" : { "hash" : BinData(0,"jtEOgd4f9yaEOeqXJ2Rjdt6Jgj8="), "keyId" : NumberLong("7670265726082482200") } }, "operationTime" : Timestamp(1785873256, 9) } bye + cat /tmp/tmp.GvWx7cRn9T + rm /tmp/tmp.wFHmcDn2Z7 /tmp/tmp.GvWx7cRn9T + return 0 + run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=sh.enableSharding("myApp2","rs2")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T8INE3dBaE +++ mktemp ++ local LAST_ERR=/tmp/tmp.gWHprkON6z ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.T8INE3dBaE ++ cat /tmp/tmp.gWHprkON6z ++ rm /tmp/tmp.T8INE3dBaE /tmp/tmp.gWHprkON6z ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.VMCHCt7oKq ++ mktemp + local LAST_ERR=/tmp/tmp.l7DArpmC2a + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VMCHCt7oKq Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bf635c7e-8239-40be-b7c1-8b080a488aa3") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1785873258, 8), "signature" : { "hash" : BinData(0,"btKk8CZ+AxyyrO8/p7gnwmM4yJU="), "keyId" : NumberLong("7670265726082482200") } }, "operationTime" : Timestamp(1785873258, 5) } bye + cat /tmp/tmp.l7DArpmC2a + rm /tmp/tmp.VMCHCt7oKq /tmp/tmp.l7DArpmC2a + return 0 + insert_data_mongos 100500 myApp '' 27019 + local data=100500 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z4s7liGIvk +++ mktemp ++ local LAST_ERR=/tmp/tmp.WREX9GNYZz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.z4s7liGIvk ++ cat /tmp/tmp.WREX9GNYZz ++ rm /tmp/tmp.z4s7liGIvk /tmp/tmp.WREX9GNYZz ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.jpzU857vo4 ++ mktemp + local LAST_ERR=/tmp/tmp.ZbJGdXEzHM + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jpzU857vo4 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("713bcae1-8f37-4321-abaa-af1027ea644e") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ZbJGdXEzHM + rm /tmp/tmp.jpzU857vo4 /tmp/tmp.ZbJGdXEzHM + return 0 + insert_data_mongos 100500 myApp1 '' 27019 + local data=100500 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ul3qeB5f0m +++ mktemp ++ local LAST_ERR=/tmp/tmp.mVpwTdKY2a ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ul3qeB5f0m ++ cat /tmp/tmp.mVpwTdKY2a ++ rm /tmp/tmp.Ul3qeB5f0m /tmp/tmp.mVpwTdKY2a ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ebv5uNDNXW ++ mktemp + local LAST_ERR=/tmp/tmp.LsHXHqEaxs + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ebv5uNDNXW Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("4d6bbdee-6a1f-4540-a57d-bbdaa0f273fc") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.LsHXHqEaxs + rm /tmp/tmp.ebv5uNDNXW /tmp/tmp.LsHXHqEaxs + return 0 + insert_data_mongos 100500 myApp2 '' 27019 + local data=100500 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PPKYEFMX3s +++ mktemp ++ local LAST_ERR=/tmp/tmp.CGfoDNru2F ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PPKYEFMX3s ++ cat /tmp/tmp.CGfoDNru2F ++ rm /tmp/tmp.PPKYEFMX3s /tmp/tmp.CGfoDNru2F ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9xye0PZ50t ++ mktemp + local LAST_ERR=/tmp/tmp.QKsnFfiID8 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9xye0PZ50t Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("41e099be-0415-44f0-8ec1-2b7d52b6f68a") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.QKsnFfiID8 + rm /tmp/tmp.9xye0PZ50t /tmp/tmp.QKsnFfiID8 + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' '' 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T19:54:25+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb '' '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VtLQsW29OE +++ mktemp ++ local LAST_ERR=/tmp/tmp.KUVnl5jD8Y ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VtLQsW29OE ++ cat /tmp/tmp.KUVnl5jD8Y ++ rm /tmp/tmp.VtLQsW29OE /tmp/tmp.KUVnl5jD8Y ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.7GERRomM98 ++ mktemp + local LAST_ERR=/tmp/tmp.iOyrVeziaR + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7GERRomM98 + cat /tmp/tmp.iOyrVeziaR + rm /tmp/tmp.7GERRomM98 /tmp/tmp.iOyrVeziaR + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2026-08-04T19:48:56.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2026-08-04T19:49:01.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2026-08-04T19:49:05.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2026-08-04T19:48:56.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2026-08-04T19:49:01.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2026-08-04T19:49:05.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2026-08-04T19:48:59.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2026-08-04T19:49:02.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2026-08-04T19:49:06.000+0000 I listening for the commands + backup_name_aws=backup-aws-s3 + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + backup_name_azure=backup-azure-blob + backup_name_oss_native=backup-oss-native + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + log 'running backup backup-minio' + set +o xtrace [2026-08-04T19:54:39+0000] running backup backup-minio + yq eval $'.metadata.name = "backup-minio"\n\t\t\t| .spec.storageName = "minio"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/backup-minio.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.W1xBcURnUY ++ mktemp + local LAST_ERR=/tmp/tmp.mSqHGhMR53 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W1xBcURnUY perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.mSqHGhMR53 + rm /tmp/tmp.W1xBcURnUY /tmp/tmp.mSqHGhMR53 + return 0 + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + log 'running backup backup-aws-s3' + set +o xtrace [2026-08-04T19:54:40+0000] running backup backup-aws-s3 + yq eval $'.metadata.name = "backup-aws-s3"\n\t\t\t| .spec.storageName = "aws-s3"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.yMdBPEnCrA ++ mktemp + local LAST_ERR=/tmp/tmp.W7Y9mgKeYD + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yMdBPEnCrA perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.W7Y9mgKeYD + rm /tmp/tmp.yMdBPEnCrA /tmp/tmp.W7Y9mgKeYD + return 0 + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + log 'running backup backup-gcp-cs' + set +o xtrace [2026-08-04T19:54:42+0000] running backup backup-gcp-cs + yq eval $'.metadata.name = "backup-gcp-cs"\n\t\t\t| .spec.storageName = "gcp-cs"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.b3y53XRNZo ++ mktemp + local LAST_ERR=/tmp/tmp.FAwlPkwg9d + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b3y53XRNZo perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.FAwlPkwg9d + rm /tmp/tmp.b3y53XRNZo /tmp/tmp.FAwlPkwg9d + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + log 'running backup backup-azure-blob' + set +o xtrace [2026-08-04T19:54:45+0000] running backup backup-azure-blob + yq eval $'.metadata.name = "backup-azure-blob"\n\t\t\t| .spec.storageName = "azure-blob"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.iAOZxBSSm0 ++ mktemp + local LAST_ERR=/tmp/tmp.NYUTwt1QxC + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iAOZxBSSm0 perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.NYUTwt1QxC + rm /tmp/tmp.iAOZxBSSm0 /tmp/tmp.NYUTwt1QxC + return 0 + run_backup oss-native + local storage=oss-native + local backup_name=backup-oss-native + local type=logical + log 'running backup backup-oss-native' + set +o xtrace [2026-08-04T19:54:47+0000] running backup backup-oss-native + yq eval $'.metadata.name = "backup-oss-native"\n\t\t\t| .spec.storageName = "oss-native"\n\t\t\t| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/backup-oss-native.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.b2I0lPSZns ++ mktemp + local LAST_ERR=/tmp/tmp.FpB26AD6XN + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.b2I0lPSZns perconaservermongodbbackup.psmdb.percona.com/backup-oss-native created + cat /tmp/tmp.FpB26AD6XN + rm /tmp/tmp.b2I0lPSZns /tmp/tmp.FpB26AD6XN + return 0 + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state..............................................................OK + wait_backup backup-gcp-cs + local backup_name=backup-gcp-cs + local target_state=ready + set +o xtrace waiting for backup-gcp-cs to reach ready state...................OK + wait_backup backup-azure-blob + local backup_name=backup-azure-blob + local target_state=ready + set +o xtrace waiting for backup-azure-blob to reach ready state..OK + wait_backup backup-oss-native + local backup_name=backup-oss-native + local target_state=ready + set +o xtrace waiting for backup-oss-native to reach ready state..OK + wait_backup backup-minio + local backup_name=backup-minio + local target_state=ready + set +o xtrace waiting for backup-minio to reach ready state..OK + sleep 5 + '[' -z '' ']' + desc 'check backup and restore -- aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- aws-s3 ----------------------------------------------------------------------------------- ++ get_backup_dest backup-aws-s3 ++ local backup_name=backup-aws-s3 ++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oci://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.ctkap2dTsl +++ mktemp ++ local LAST_ERR=/tmp/tmp.m3v4gsoRoC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-aws-s3' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ctkap2dTsl ++ cat /tmp/tmp.m3v4gsoRoC ++ rm /tmp/tmp.ctkap2dTsl /tmp/tmp.m3v4gsoRoC ++ return 0 + backup_dest_aws=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_aws_credentials + local secret_name=aws-s3-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded/2026-08-04T19:56:22Z + key=/rs0/myApp.test.gz + endpoint= + storage_name=aws-s3 + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n '' ]] + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs0/myApp.test.gz found in aws-s3' Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs0/myApp.test.gz found in aws-s3 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z /rs1/myApp1.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded/2026-08-04T19:56:22Z + key=/rs1/myApp1.test.gz + endpoint= + storage_name=aws-s3 + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n '' ]] + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs1/myApp1.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs1/myApp1.test.gz found in aws-s3' Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs1/myApp1.test.gz found in aws-s3 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z /rs2/myApp2.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded/2026-08-04T19:56:22Z + key=/rs2/myApp2.test.gz + endpoint= + storage_name=aws-s3 + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n '' ]] + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs2/myApp2.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs2/myApp2.test.gz found in aws-s3' Backup operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z//rs2/myApp2.test.gz found in aws-s3 + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + insert_data_mongos 100501 myApp '' 27019 + local data=100501 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.znzUQTmSr6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2RZqjnxQRu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.znzUQTmSr6 ++ cat /tmp/tmp.2RZqjnxQRu ++ rm /tmp/tmp.znzUQTmSr6 /tmp/tmp.2RZqjnxQRu ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.XTuwcxKgdx ++ mktemp + local LAST_ERR=/tmp/tmp.1sSLDYMSnm + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XTuwcxKgdx Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("11f88244-2d0b-426e-96fc-681c6f3ca68c") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.1sSLDYMSnm + rm /tmp/tmp.XTuwcxKgdx /tmp/tmp.1sSLDYMSnm + return 0 + insert_data_mongos 100501 myApp1 '' 27019 + local data=100501 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QEx2Ym28aF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wi8zZtjN6L ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QEx2Ym28aF ++ cat /tmp/tmp.Wi8zZtjN6L ++ rm /tmp/tmp.QEx2Ym28aF /tmp/tmp.Wi8zZtjN6L ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.uCXOjPHCZF ++ mktemp + local LAST_ERR=/tmp/tmp.cWsxofq7hv + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uCXOjPHCZF Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("d99ece26-55b7-4c7e-99a7-df5db2689654") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.cWsxofq7hv + rm /tmp/tmp.uCXOjPHCZF /tmp/tmp.cWsxofq7hv + return 0 + insert_data_mongos 100501 myApp2 '' 27019 + local data=100501 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4R2HN4fOyr +++ mktemp ++ local LAST_ERR=/tmp/tmp.KuMWezTbNg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4R2HN4fOyr ++ cat /tmp/tmp.KuMWezTbNg ++ rm /tmp/tmp.4R2HN4fOyr /tmp/tmp.KuMWezTbNg ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.7ZsOZ3HxES ++ mktemp + local LAST_ERR=/tmp/tmp.IevXF6RVog + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7ZsOZ3HxES Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("4f8b7ae4-1147-4e9a-a7aa-952218683367") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.IevXF6RVog + rm /tmp/tmp.7ZsOZ3HxES /tmp/tmp.IevXF6RVog + return 0 + check_data -2nd + local postfix=-2nd ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T19:57:52+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hCQ5r4SFp8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.v5G4VQi8t6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hCQ5r4SFp8 ++ cat /tmp/tmp.v5G4VQi8t6 ++ rm /tmp/tmp.hCQ5r4SFp8 /tmp/tmp.v5G4VQi8t6 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.LEypoyI3t6 ++ mktemp + local LAST_ERR=/tmp/tmp.ErgEBN2E6X + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LEypoyI3t6 + cat /tmp/tmp.ErgEBN2E6X + rm /tmp/tmp.LEypoyI3t6 /tmp/tmp.ErgEBN2E6X + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.hfrg7he8cN/find-2nd + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T19:57:54+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FgIuXsLaf5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IHBR01LkSM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FgIuXsLaf5 ++ cat /tmp/tmp.IHBR01LkSM ++ rm /tmp/tmp.FgIuXsLaf5 /tmp/tmp.IHBR01LkSM ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.p03aBRf6Tq ++ mktemp + local LAST_ERR=/tmp/tmp.w64lFOnQa8 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.p03aBRf6Tq + cat /tmp/tmp.w64lFOnQa8 + rm /tmp/tmp.p03aBRf6Tq /tmp/tmp.w64lFOnQa8 + return 0 + compare_mongo_output find -2nd1 + local command=find + local postfix=-2nd1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.hfrg7he8cN/find-2nd1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T19:57:56+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp2\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kRkLFTUbtU +++ mktemp ++ local LAST_ERR=/tmp/tmp.m93Ml4UEL1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kRkLFTUbtU ++ cat /tmp/tmp.m93Ml4UEL1 ++ rm /tmp/tmp.kRkLFTUbtU /tmp/tmp.m93Ml4UEL1 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.7IOZJvoSwe ++ mktemp + local LAST_ERR=/tmp/tmp.akDi9t3j5v + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7IOZJvoSwe + cat /tmp/tmp.akDi9t3j5v + rm /tmp/tmp.7IOZJvoSwe /tmp/tmp.akDi9t3j5v + return 0 + compare_mongo_output find -2nd2 + local command=find + local postfix=-2nd2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.hfrg7he8cN/find-2nd2 + return + run_restore backup-aws-s3 + local backup_name=backup-aws-s3 + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + log 'running restore restore-backup-aws-s3' + set +o xtrace [2026-08-04T19:57:58+0000] running restore restore-backup-aws-s3 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.fIf6o6OVtd ++ mktemp + local LAST_ERR=/tmp/tmp.k1mlTu6gXu + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fIf6o6OVtd perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created + cat /tmp/tmp.k1mlTu6gXu + rm /tmp/tmp.fIf6o6OVtd /tmp/tmp.k1mlTu6gXu + return 0 + wait_restore backup-aws-s3 some-name + local backup_name=backup-aws-s3 + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.quI252xRk3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RK32H2yO4h ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.quI252xRk3 ++ cat /tmp/tmp.RK32H2yO4h ++ rm /tmp/tmp.quI252xRk3 /tmp/tmp.RK32H2yO4h ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xsWmaDwMjz +++ mktemp ++ local LAST_ERR=/tmp/tmp.NLp7erNa7W ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xsWmaDwMjz ++ cat /tmp/tmp.NLp7erNa7W ++ rm /tmp/tmp.xsWmaDwMjz /tmp/tmp.NLp7erNa7W ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HlCMaralds +++ mktemp ++ local LAST_ERR=/tmp/tmp.r5a4CZhLNh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HlCMaralds ++ cat /tmp/tmp.r5a4CZhLNh ++ rm /tmp/tmp.HlCMaralds /tmp/tmp.r5a4CZhLNh ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FHiflXY2Yo +++ mktemp ++ local LAST_ERR=/tmp/tmp.eEoWUdwZ1v ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FHiflXY2Yo ++ cat /tmp/tmp.eEoWUdwZ1v ++ rm /tmp/tmp.FHiflXY2Yo /tmp/tmp.eEoWUdwZ1v ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.A3WPTekO0D +++ mktemp ++ local LAST_ERR=/tmp/tmp.cSaOc6tfGQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.A3WPTekO0D ++ cat /tmp/tmp.cSaOc6tfGQ ++ rm /tmp/tmp.A3WPTekO0D /tmp/tmp.cSaOc6tfGQ ++ return 0 + [[ ready == ready ]] + echo .OK .OK + check_data + local postfix= ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:00:23+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BgUyDpX5bo +++ mktemp ++ local LAST_ERR=/tmp/tmp.9Bhwy5odUG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BgUyDpX5bo ++ cat /tmp/tmp.9Bhwy5odUG ++ rm /tmp/tmp.BgUyDpX5bo /tmp/tmp.9Bhwy5odUG ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.zxaFFDWLbg ++ mktemp + local LAST_ERR=/tmp/tmp.nAZLM4Bq2X + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zxaFFDWLbg + cat /tmp/tmp.nAZLM4Bq2X + rm /tmp/tmp.zxaFFDWLbg /tmp/tmp.nAZLM4Bq2X + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:00:26+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp1\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Opf7s0oype +++ mktemp ++ local LAST_ERR=/tmp/tmp.kwaGfCccD0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Opf7s0oype ++ cat /tmp/tmp.kwaGfCccD0 ++ rm /tmp/tmp.Opf7s0oype /tmp/tmp.kwaGfCccD0 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Qx2miT6qnZ ++ mktemp + local LAST_ERR=/tmp/tmp.a4xxOyOIW6 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Qx2miT6qnZ + cat /tmp/tmp.a4xxOyOIW6 + rm /tmp/tmp.Qx2miT6qnZ /tmp/tmp.a4xxOyOIW6 + return 0 + compare_mongo_output find 1 + local command=find + local postfix=1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.hfrg7he8cN/find1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:00:28+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OVnGk6HhBU +++ mktemp ++ local LAST_ERR=/tmp/tmp.V7gBKTeHgS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OVnGk6HhBU ++ cat /tmp/tmp.V7gBKTeHgS ++ rm /tmp/tmp.OVnGk6HhBU /tmp/tmp.V7gBKTeHgS ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.zQEI3hlleS ++ mktemp + local LAST_ERR=/tmp/tmp.XScc3wV46b + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zQEI3hlleS + cat /tmp/tmp.XScc3wV46b + rm /tmp/tmp.zQEI3hlleS /tmp/tmp.XScc3wV46b + return 0 + compare_mongo_output find 2 + local command=find + local postfix=2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.hfrg7he8cN/find2 + return + desc 'check backup and restore -- gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- gcp-cs ----------------------------------------------------------------------------------- ++ get_backup_dest backup-gcp-cs ++ local backup_name=backup-gcp-cs ++ kubectl_bin get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.w0EcahGGqe +++ mktemp ++ local LAST_ERR=/tmp/tmp.fYrtqPHH1j ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-gcp-cs' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.w0EcahGGqe ++ cat /tmp/tmp.fYrtqPHH1j ++ rm /tmp/tmp.w0EcahGGqe /tmp/tmp.fYrtqPHH1j ++ return 0 + backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z + setup_gcs_credentials + local secret_name=gcp-cs-secret + gsutil ls + echo 'Setting up GCS credentials from K8s secret: gcp-cs-secret' Setting up GCS credentials from K8s secret: gcp-cs-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'GCS credentials configured successfully' GCS credentials configured successfully + check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz ++ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz + gsutil_err=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz found in gcp-cs + check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z /rs1/myApp1.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z + obj=/rs1/myApp1.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs1/myApp1.test.gz ++ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs1/myApp1.test.gz + gsutil_err=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs1/myApp1.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs1/myApp1.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs1/myApp1.test.gz found in gcp-cs + check_backup_existence_gcs operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z /rs2/myApp2.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z + obj=/rs2/myApp2.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs2/myApp2.test.gz ++ gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs2/myApp2.test.gz + gsutil_err=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs2/myApp2.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs2/myApp2.test.gz found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs2/myApp2.test.gz found in gcp-cs + insert_data_mongos 100501 myApp '' 27019 + local data=100501 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.boMyYwSjzF +++ mktemp ++ local LAST_ERR=/tmp/tmp.KBBIQu9i8G ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.boMyYwSjzF ++ cat /tmp/tmp.KBBIQu9i8G ++ rm /tmp/tmp.boMyYwSjzF /tmp/tmp.KBBIQu9i8G ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.1OhqYlyYXI ++ mktemp + local LAST_ERR=/tmp/tmp.2mrcS73NW9 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1OhqYlyYXI Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("21edb237-8f04-48d0-a6da-67e256e2f1db") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2mrcS73NW9 + rm /tmp/tmp.1OhqYlyYXI /tmp/tmp.2mrcS73NW9 + return 0 + insert_data_mongos 100501 myApp1 '' 27019 + local data=100501 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yTAKHyB84K +++ mktemp ++ local LAST_ERR=/tmp/tmp.X7r2BShR01 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yTAKHyB84K ++ cat /tmp/tmp.X7r2BShR01 ++ rm /tmp/tmp.yTAKHyB84K /tmp/tmp.X7r2BShR01 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.hIQDxIXXjd ++ mktemp + local LAST_ERR=/tmp/tmp.LdbXlgMWNr + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hIQDxIXXjd Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("572e957a-9be5-48be-9f41-9a333e1be268") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.LdbXlgMWNr + rm /tmp/tmp.hIQDxIXXjd /tmp/tmp.LdbXlgMWNr + return 0 + insert_data_mongos 100501 myApp2 '' 27019 + local data=100501 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gmiyUqFkAT +++ mktemp ++ local LAST_ERR=/tmp/tmp.GrpjNlMeDU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gmiyUqFkAT ++ cat /tmp/tmp.GrpjNlMeDU ++ rm /tmp/tmp.gmiyUqFkAT /tmp/tmp.GrpjNlMeDU ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.A6kf107YzD ++ mktemp + local LAST_ERR=/tmp/tmp.ShJFYBQfOD + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.A6kf107YzD Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("aaafdc3c-2f42-4e17-b9b1-fb56725d9b00") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.ShJFYBQfOD + rm /tmp/tmp.A6kf107YzD /tmp/tmp.ShJFYBQfOD + return 0 + check_data -2nd + local postfix=-2nd ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:00:47+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2HlcgS9H5l +++ mktemp ++ local LAST_ERR=/tmp/tmp.fT3vHEzSPm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2HlcgS9H5l ++ cat /tmp/tmp.fT3vHEzSPm ++ rm /tmp/tmp.2HlcgS9H5l /tmp/tmp.fT3vHEzSPm ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.l5ZneQhNHv ++ mktemp + local LAST_ERR=/tmp/tmp.YD6yVnJrP3 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l5ZneQhNHv + cat /tmp/tmp.YD6yVnJrP3 + rm /tmp/tmp.l5ZneQhNHv /tmp/tmp.YD6yVnJrP3 + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.hfrg7he8cN/find-2nd + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:00:50+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PAaDEqrLgt +++ mktemp ++ local LAST_ERR=/tmp/tmp.X7R9cL6aRQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PAaDEqrLgt ++ cat /tmp/tmp.X7R9cL6aRQ ++ rm /tmp/tmp.PAaDEqrLgt /tmp/tmp.X7R9cL6aRQ ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.AypkhzP8A6 ++ mktemp + local LAST_ERR=/tmp/tmp.gquWYyAm7K + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AypkhzP8A6 + cat /tmp/tmp.gquWYyAm7K + rm /tmp/tmp.AypkhzP8A6 /tmp/tmp.gquWYyAm7K + return 0 + compare_mongo_output find -2nd1 + local command=find + local postfix=-2nd1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.hfrg7he8cN/find-2nd1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:00:53+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q1Db1ClVn5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.zzQrlGHG7o ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.q1Db1ClVn5 ++ cat /tmp/tmp.zzQrlGHG7o ++ rm /tmp/tmp.q1Db1ClVn5 /tmp/tmp.zzQrlGHG7o ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.qJTtNmteOn ++ mktemp + local LAST_ERR=/tmp/tmp.fWlLMWVcwS + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qJTtNmteOn + cat /tmp/tmp.fWlLMWVcwS + rm /tmp/tmp.qJTtNmteOn /tmp/tmp.fWlLMWVcwS + return 0 + compare_mongo_output find -2nd2 + local command=find + local postfix=-2nd2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.hfrg7he8cN/find-2nd2 + return + run_restore backup-gcp-cs + local backup_name=backup-gcp-cs + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + log 'running restore restore-backup-gcp-cs' + set +o xtrace [2026-08-04T20:00:55+0000] running restore restore-backup-gcp-cs + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.t96AR2TO84 ++ mktemp + local LAST_ERR=/tmp/tmp.osRRgvuvkH + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t96AR2TO84 perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created + cat /tmp/tmp.osRRgvuvkH + rm /tmp/tmp.t96AR2TO84 /tmp/tmp.osRRgvuvkH + return 0 + wait_restore backup-gcp-cs some-name + local backup_name=backup-gcp-cs + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-gcp-cs object to be created.OK Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dclwpivaDl +++ mktemp ++ local LAST_ERR=/tmp/tmp.KWV0xlDG8q ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dclwpivaDl ++ cat /tmp/tmp.KWV0xlDG8q ++ rm /tmp/tmp.dclwpivaDl /tmp/tmp.KWV0xlDG8q ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j6g1fQOHz8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9JA50LrmSg ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.j6g1fQOHz8 ++ cat /tmp/tmp.9JA50LrmSg ++ rm /tmp/tmp.j6g1fQOHz8 /tmp/tmp.9JA50LrmSg ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QVnWdIdefy +++ mktemp ++ local LAST_ERR=/tmp/tmp.xgiz9FrJEd ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QVnWdIdefy ++ cat /tmp/tmp.xgiz9FrJEd ++ rm /tmp/tmp.QVnWdIdefy /tmp/tmp.xgiz9FrJEd ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rVIE7JtKt3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.NmGVa65oHP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rVIE7JtKt3 ++ cat /tmp/tmp.NmGVa65oHP ++ rm /tmp/tmp.rVIE7JtKt3 /tmp/tmp.NmGVa65oHP ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tRcUkEYCdT +++ mktemp ++ local LAST_ERR=/tmp/tmp.BHT1Y9UZHA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tRcUkEYCdT ++ cat /tmp/tmp.BHT1Y9UZHA ++ rm /tmp/tmp.tRcUkEYCdT /tmp/tmp.BHT1Y9UZHA ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 5 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sJYvkdZpKj +++ mktemp ++ local LAST_ERR=/tmp/tmp.avV1PYldho ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sJYvkdZpKj ++ cat /tmp/tmp.avV1PYldho ++ rm /tmp/tmp.sJYvkdZpKj /tmp/tmp.avV1PYldho ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 6 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.58BUO4Mt5U +++ mktemp ++ local LAST_ERR=/tmp/tmp.jCQH2KgaCz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.58BUO4Mt5U ++ cat /tmp/tmp.jCQH2KgaCz ++ rm /tmp/tmp.58BUO4Mt5U /tmp/tmp.jCQH2KgaCz ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 7 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1NEA32HecV +++ mktemp ++ local LAST_ERR=/tmp/tmp.UTYvyvHJIC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1NEA32HecV ++ cat /tmp/tmp.UTYvyvHJIC ++ rm /tmp/tmp.1NEA32HecV /tmp/tmp.UTYvyvHJIC ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 8 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.I6kZxNJ6qI +++ mktemp ++ local LAST_ERR=/tmp/tmp.nPItxGlnxZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.I6kZxNJ6qI ++ cat /tmp/tmp.nPItxGlnxZ ++ rm /tmp/tmp.I6kZxNJ6qI /tmp/tmp.nPItxGlnxZ ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 9 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZWOkOgpvjE +++ mktemp ++ local LAST_ERR=/tmp/tmp.9v6AvTu2KN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZWOkOgpvjE ++ cat /tmp/tmp.9v6AvTu2KN ++ rm /tmp/tmp.ZWOkOgpvjE /tmp/tmp.9v6AvTu2KN ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 10 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tHgDoztvhY +++ mktemp ++ local LAST_ERR=/tmp/tmp.w2zT4jKXUp ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tHgDoztvhY ++ cat /tmp/tmp.w2zT4jKXUp ++ rm /tmp/tmp.tHgDoztvhY /tmp/tmp.w2zT4jKXUp ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 11 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EGlvYffeVP +++ mktemp ++ local LAST_ERR=/tmp/tmp.bUQmtgWQZu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EGlvYffeVP ++ cat /tmp/tmp.bUQmtgWQZu ++ rm /tmp/tmp.EGlvYffeVP /tmp/tmp.bUQmtgWQZu ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 12 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.N79V4FORZX +++ mktemp ++ local LAST_ERR=/tmp/tmp.P8n9SeDrNz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N79V4FORZX ++ cat /tmp/tmp.P8n9SeDrNz ++ rm /tmp/tmp.N79V4FORZX /tmp/tmp.P8n9SeDrNz ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 13 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.i1T7njxWsr +++ mktemp ++ local LAST_ERR=/tmp/tmp.SUM9PGXOuB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.i1T7njxWsr ++ cat /tmp/tmp.SUM9PGXOuB ++ rm /tmp/tmp.i1T7njxWsr /tmp/tmp.SUM9PGXOuB ++ return 0 + [[ ready == ready ]] + echo .OK .OK + check_data + local postfix= ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:04:59+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CGWKzfAt6y +++ mktemp ++ local LAST_ERR=/tmp/tmp.LyHc6OVkAG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CGWKzfAt6y ++ cat /tmp/tmp.LyHc6OVkAG ++ rm /tmp/tmp.CGWKzfAt6y /tmp/tmp.LyHc6OVkAG ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.bCWcFp6NIT ++ mktemp + local LAST_ERR=/tmp/tmp.Q1xViVT0ae + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bCWcFp6NIT + cat /tmp/tmp.Q1xViVT0ae + rm /tmp/tmp.bCWcFp6NIT /tmp/tmp.Q1xViVT0ae + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:05:02+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hbXGXKG9ax +++ mktemp ++ local LAST_ERR=/tmp/tmp.YHRXNFTDvF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hbXGXKG9ax ++ cat /tmp/tmp.YHRXNFTDvF ++ rm /tmp/tmp.hbXGXKG9ax /tmp/tmp.YHRXNFTDvF ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.eQ3J39iCq9 ++ mktemp + local LAST_ERR=/tmp/tmp.FP4OahVPnK + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eQ3J39iCq9 + cat /tmp/tmp.FP4OahVPnK + rm /tmp/tmp.eQ3J39iCq9 /tmp/tmp.FP4OahVPnK + return 0 + compare_mongo_output find 1 + local command=find + local postfix=1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.hfrg7he8cN/find1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:05:04+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r5I9PBJ4wM +++ mktemp ++ local LAST_ERR=/tmp/tmp.V89zod2dTn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r5I9PBJ4wM ++ cat /tmp/tmp.V89zod2dTn ++ rm /tmp/tmp.r5I9PBJ4wM /tmp/tmp.V89zod2dTn ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.FZFcFfU4gE ++ mktemp + local LAST_ERR=/tmp/tmp.mpWoJqDAJv + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FZFcFfU4gE + cat /tmp/tmp.mpWoJqDAJv + rm /tmp/tmp.FZFcFfU4gE /tmp/tmp.mpWoJqDAJv + return 0 + compare_mongo_output find 2 + local command=find + local postfix=2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.hfrg7he8cN/find2 + return + desc 'check backup and restore -- azure-blob' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- azure-blob ----------------------------------------------------------------------------------- ++ get_backup_dest backup-azure-blob ++ local backup_name=backup-azure-blob ++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.2WmEWd4F5l ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.HSvM4X8Kpi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-azure-blob' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2WmEWd4F5l ++ cat /tmp/tmp.HSvM4X8Kpi ++ rm /tmp/tmp.2WmEWd4F5l /tmp/tmp.HSvM4X8Kpi ++ return 0 + backup_dest_azure=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + setup_azure_credentials + local secret_name=azure-secret + echo 'Setting up Azure credentials from K8s secret: azure-secret' Setting up Azure credentials from K8s secret: azure-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'Azure credentials configured successfully' Azure credentials configured successfully + check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz found in azure-blob' Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz found in azure-blob + check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z /rs1/myApp1.test.gz ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + url_path=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f1 + container=operator-testing ++ cut -d/ -f2- ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + blob_prefix=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + blob=/rs1/myApp1.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs1/myApp1.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs1/myApp1.test.gz + echo 'Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs1/myApp1.test.gz found in azure-blob' Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs1/myApp1.test.gz found in azure-blob + check_backup_existence_azure operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z /rs2/myApp2.test.gz ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + url_path=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + blob=/rs2/myApp2.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs2/myApp2.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs2/myApp2.test.gz + echo 'Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs2/myApp2.test.gz found in azure-blob' Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs2/myApp2.test.gz found in azure-blob + insert_data_mongos 100501 myApp '' 27019 + local data=100501 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0VOgiwv8op +++ mktemp ++ local LAST_ERR=/tmp/tmp.vPl3X0Beb4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0VOgiwv8op ++ cat /tmp/tmp.vPl3X0Beb4 ++ rm /tmp/tmp.0VOgiwv8op /tmp/tmp.vPl3X0Beb4 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.VyIASlxPVk ++ mktemp + local LAST_ERR=/tmp/tmp.TQwPdycfxj + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.VyIASlxPVk Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("6a24d0f7-75b3-4e22-9600-fa346978cdbe") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.TQwPdycfxj + rm /tmp/tmp.VyIASlxPVk /tmp/tmp.TQwPdycfxj + return 0 + insert_data_mongos 100501 myApp1 '' 27019 + local data=100501 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GDZxNRYMXv +++ mktemp ++ local LAST_ERR=/tmp/tmp.8BUJVpKl6e ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GDZxNRYMXv ++ cat /tmp/tmp.8BUJVpKl6e ++ rm /tmp/tmp.GDZxNRYMXv /tmp/tmp.8BUJVpKl6e ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.oRutXg76D9 ++ mktemp + local LAST_ERR=/tmp/tmp.GrorzVj1Sx + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oRutXg76D9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("6389ab03-6b8e-49bc-851f-7e13397e9984") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.GrorzVj1Sx + rm /tmp/tmp.oRutXg76D9 /tmp/tmp.GrorzVj1Sx + return 0 + insert_data_mongos 100501 myApp2 '' 27019 + local data=100501 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ujqeOLivI4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.46hqp9vCt9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ujqeOLivI4 ++ cat /tmp/tmp.46hqp9vCt9 ++ rm /tmp/tmp.ujqeOLivI4 /tmp/tmp.46hqp9vCt9 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.kclNs5enGQ ++ mktemp + local LAST_ERR=/tmp/tmp.kf2yAlF265 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kclNs5enGQ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("5cd0df78-4c13-4607-a55c-ceac38e15f7b") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.kf2yAlF265 + rm /tmp/tmp.kclNs5enGQ /tmp/tmp.kf2yAlF265 + return 0 + check_data -2nd + local postfix=-2nd ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:05:20+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find()' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V1bbcIAKEq +++ mktemp ++ local LAST_ERR=/tmp/tmp.ccZRaH2hVz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V1bbcIAKEq ++ cat /tmp/tmp.ccZRaH2hVz ++ rm /tmp/tmp.V1bbcIAKEq /tmp/tmp.ccZRaH2hVz ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.rJ0R8yZWfg ++ mktemp + local LAST_ERR=/tmp/tmp.Rf4Y2gMyQo + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rJ0R8yZWfg + cat /tmp/tmp.Rf4Y2gMyQo + rm /tmp/tmp.rJ0R8yZWfg /tmp/tmp.Rf4Y2gMyQo + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.hfrg7he8cN/find-2nd + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:05:23+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XD66ACn8fV +++ mktemp ++ local LAST_ERR=/tmp/tmp.F4u26NzQex ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.XD66ACn8fV ++ cat /tmp/tmp.F4u26NzQex ++ rm /tmp/tmp.XD66ACn8fV /tmp/tmp.F4u26NzQex ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.4jVl4K68BZ ++ mktemp + local LAST_ERR=/tmp/tmp.bQYnoGwCAA + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4jVl4K68BZ + cat /tmp/tmp.bQYnoGwCAA + rm /tmp/tmp.4jVl4K68BZ /tmp/tmp.bQYnoGwCAA + return 0 + compare_mongo_output find -2nd1 + local command=find + local postfix=-2nd1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.hfrg7he8cN/find-2nd1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:05:25+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FrwJQCRqLW +++ mktemp ++ local LAST_ERR=/tmp/tmp.5HT5k4JVbB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FrwJQCRqLW ++ cat /tmp/tmp.5HT5k4JVbB ++ rm /tmp/tmp.FrwJQCRqLW /tmp/tmp.5HT5k4JVbB ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.GOLzMNSaP2 ++ mktemp + local LAST_ERR=/tmp/tmp.AVczYjD1Rj + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GOLzMNSaP2 + cat /tmp/tmp.AVczYjD1Rj + rm /tmp/tmp.GOLzMNSaP2 /tmp/tmp.AVczYjD1Rj + return 0 + compare_mongo_output find -2nd2 + local command=find + local postfix=-2nd2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.hfrg7he8cN/find-2nd2 + return + run_restore backup-azure-blob + local backup_name=backup-azure-blob + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + log 'running restore restore-backup-azure-blob' + set +o xtrace [2026-08-04T20:05:28+0000] running restore restore-backup-azure-blob + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/' + kubectl_bin apply -f - + /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/' ++ mktemp + local LAST_OUT=/tmp/tmp.fYLPLlLacD ++ mktemp + local LAST_ERR=/tmp/tmp.d8kGGHAWIj + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fYLPLlLacD perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created + cat /tmp/tmp.d8kGGHAWIj + rm /tmp/tmp.fYLPLlLacD /tmp/tmp.d8kGGHAWIj + return 0 + wait_restore backup-azure-blob some-name + local backup_name=backup-azure-blob + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r1wS7RokoZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.QKsqIgeQ6B ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r1wS7RokoZ ++ cat /tmp/tmp.QKsqIgeQ6B ++ rm /tmp/tmp.r1wS7RokoZ /tmp/tmp.QKsqIgeQ6B ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bpT73AaliZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.pyyao4bFfv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bpT73AaliZ ++ cat /tmp/tmp.pyyao4bFfv ++ rm /tmp/tmp.bpT73AaliZ /tmp/tmp.pyyao4bFfv ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EjZ240dP5n +++ mktemp ++ local LAST_ERR=/tmp/tmp.W0q0NZKjzY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EjZ240dP5n ++ cat /tmp/tmp.W0q0NZKjzY ++ rm /tmp/tmp.EjZ240dP5n /tmp/tmp.W0q0NZKjzY ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3NvI7rH0oz +++ mktemp ++ local LAST_ERR=/tmp/tmp.6l770IqhHt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3NvI7rH0oz ++ cat /tmp/tmp.6l770IqhHt ++ rm /tmp/tmp.3NvI7rH0oz /tmp/tmp.6l770IqhHt ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1Klnuu4OeV +++ mktemp ++ local LAST_ERR=/tmp/tmp.OoyjGzzZB0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1Klnuu4OeV ++ cat /tmp/tmp.OoyjGzzZB0 ++ rm /tmp/tmp.1Klnuu4OeV /tmp/tmp.OoyjGzzZB0 ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 5 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mxK9d6D97c +++ mktemp ++ local LAST_ERR=/tmp/tmp.enww1nmBmz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mxK9d6D97c ++ cat /tmp/tmp.enww1nmBmz ++ rm /tmp/tmp.mxK9d6D97c /tmp/tmp.enww1nmBmz ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 6 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f4XLdycSqn +++ mktemp ++ local LAST_ERR=/tmp/tmp.vxlBNOq1AD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f4XLdycSqn ++ cat /tmp/tmp.vxlBNOq1AD ++ rm /tmp/tmp.f4XLdycSqn /tmp/tmp.vxlBNOq1AD ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 7 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f5ZmZakfHR +++ mktemp ++ local LAST_ERR=/tmp/tmp.DkBIMIZKQb ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f5ZmZakfHR ++ cat /tmp/tmp.DkBIMIZKQb ++ rm /tmp/tmp.f5ZmZakfHR /tmp/tmp.DkBIMIZKQb ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 8 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5IVPCXxQRB +++ mktemp ++ local LAST_ERR=/tmp/tmp.jmUiZNpSK4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5IVPCXxQRB ++ cat /tmp/tmp.jmUiZNpSK4 ++ rm /tmp/tmp.5IVPCXxQRB /tmp/tmp.jmUiZNpSK4 ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 9 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lFCL4Xi6xj +++ mktemp ++ local LAST_ERR=/tmp/tmp.nYuf36eFmh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lFCL4Xi6xj ++ cat /tmp/tmp.nYuf36eFmh ++ rm /tmp/tmp.lFCL4Xi6xj /tmp/tmp.nYuf36eFmh ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 10 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Gqk59sOJdL +++ mktemp ++ local LAST_ERR=/tmp/tmp.7APnUuW4Hr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Gqk59sOJdL ++ cat /tmp/tmp.7APnUuW4Hr ++ rm /tmp/tmp.Gqk59sOJdL /tmp/tmp.7APnUuW4Hr ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 11 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mZBVeIHpVv +++ mktemp ++ local LAST_ERR=/tmp/tmp.xBICus7PtT ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mZBVeIHpVv ++ cat /tmp/tmp.xBICus7PtT ++ rm /tmp/tmp.mZBVeIHpVv /tmp/tmp.xBICus7PtT ++ return 0 + [[ ready == ready ]] + echo .OK .OK + check_data + local postfix= ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:09:01+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CfmVg8exzQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.KH2eu6PCIk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CfmVg8exzQ ++ cat /tmp/tmp.KH2eu6PCIk ++ rm /tmp/tmp.CfmVg8exzQ /tmp/tmp.KH2eu6PCIk ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ZNOMDyjZvp ++ mktemp + local LAST_ERR=/tmp/tmp.HF8jjaDUB5 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZNOMDyjZvp + cat /tmp/tmp.HF8jjaDUB5 + rm /tmp/tmp.ZNOMDyjZvp /tmp/tmp.HF8jjaDUB5 + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:09:04+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ echo .svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c7fU4FllL4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ERHfSiz1fP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c7fU4FllL4 ++ cat /tmp/tmp.ERHfSiz1fP ++ rm /tmp/tmp.c7fU4FllL4 /tmp/tmp.ERHfSiz1fP ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.UxeGGu2oSm ++ mktemp + local LAST_ERR=/tmp/tmp.O1hh7L2mql + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UxeGGu2oSm + cat /tmp/tmp.O1hh7L2mql + rm /tmp/tmp.UxeGGu2oSm /tmp/tmp.O1hh7L2mql + return 0 + compare_mongo_output find 1 + local command=find + local postfix=1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.hfrg7he8cN/find1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:09:07+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R5LI6B5uB7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.s6ELVYEGzM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.R5LI6B5uB7 ++ cat /tmp/tmp.s6ELVYEGzM ++ rm /tmp/tmp.R5LI6B5uB7 /tmp/tmp.s6ELVYEGzM ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.6cxP4tHSet ++ mktemp + local LAST_ERR=/tmp/tmp.KC7FARQfFa + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6cxP4tHSet + cat /tmp/tmp.KC7FARQfFa + rm /tmp/tmp.6cxP4tHSet /tmp/tmp.KC7FARQfFa + return 0 + compare_mongo_output find 2 + local command=find + local postfix=2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.hfrg7he8cN/find2 + return + desc 'check backup and restore -- oss-native' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- oss-native ----------------------------------------------------------------------------------- ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|azure://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.4lJvQhI1iS +++ mktemp ++ sed 's|oci://||' ++ local LAST_ERR=/tmp/tmp.SUCFXRFwpr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4lJvQhI1iS ++ cat /tmp/tmp.SUCFXRFwpr ++ rm /tmp/tmp.4lJvQhI1iS /tmp/tmp.SUCFXRFwpr ++ return 0 + backup_dest_oss_native=operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YEZ9aas0ai ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tpz3M6NrjS +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.YEZ9aas0ai +++ cat /tmp/tmp.tpz3M6NrjS +++ rm /tmp/tmp.YEZ9aas0ai /tmp/tmp.tpz3M6NrjS +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iQDIe6esxr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KV3sQLAffC +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.iQDIe6esxr +++ cat /tmp/tmp.KV3sQLAffC +++ rm /tmp/tmp.iQDIe6esxr /tmp/tmp.KV3sQLAffC +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0SaobwSFS2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9pVP8gYuEw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0SaobwSFS2 ++ cat /tmp/tmp.9pVP8gYuEw ++ rm /tmp/tmp.0SaobwSFS2 /tmp/tmp.9pVP8gYuEw ++ return 0 + backup_endpoint_oss_native=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DlbNdiyLnb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hpUX0ZqEcy +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.DlbNdiyLnb +++ cat /tmp/tmp.hpUX0ZqEcy +++ rm /tmp/tmp.DlbNdiyLnb /tmp/tmp.hpUX0ZqEcy +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JTgFc6Zev3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MSpbg8zJiT +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.JTgFc6Zev3 +++ cat /tmp/tmp.MSpbg8zJiT +++ rm /tmp/tmp.JTgFc6Zev3 /tmp/tmp.MSpbg8zJiT +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6AmGjZYhTX +++ mktemp ++ local LAST_ERR=/tmp/tmp.JrBIImtgj0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6AmGjZYhTX ++ cat /tmp/tmp.JrBIImtgj0 ++ rm /tmp/tmp.6AmGjZYhTX /tmp/tmp.JrBIImtgj0 ++ return 0 + backup_region_oss_native=eu-central-1 + check_backup_in_storage backup-oss-native oss rs0 myApp.test.gz + local backup=backup-oss-native + local storage_type=oss + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ sed 's|oss://||' ++ sed 's|oci://||' ++ local LAST_OUT=/tmp/tmp.goXMVWdo3Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.GsdNjpdhEV ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.goXMVWdo3Q ++ cat /tmp/tmp.GsdNjpdhEV ++ rm /tmp/tmp.goXMVWdo3Q /tmp/tmp.GsdNjpdhEV ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ICeQqSQYLd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RuYzWt2fAI +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ICeQqSQYLd +++ cat /tmp/tmp.RuYzWt2fAI +++ rm /tmp/tmp.ICeQqSQYLd /tmp/tmp.RuYzWt2fAI +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lKyF9pvLwO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OYp3c3Aqk7 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.lKyF9pvLwO +++ cat /tmp/tmp.OYp3c3Aqk7 +++ rm /tmp/tmp.lKyF9pvLwO /tmp/tmp.OYp3c3Aqk7 +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fvGiqFW3y6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MZX6vXQHv4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fvGiqFW3y6 ++ cat /tmp/tmp.MZX6vXQHv4 ++ rm /tmp/tmp.fvGiqFW3y6 /tmp/tmp.MZX6vXQHv4 ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.q5dP3QCQI3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LrAjFGU6kl +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.q5dP3QCQI3 +++ cat /tmp/tmp.LrAjFGU6kl +++ rm /tmp/tmp.q5dP3QCQI3 /tmp/tmp.LrAjFGU6kl +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.71U9jGlwdm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CDgHXgqC3e +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.71U9jGlwdm +++ cat /tmp/tmp.CDgHXgqC3e +++ rm /tmp/tmp.71U9jGlwdm /tmp/tmp.CDgHXgqC3e +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xMVZdE431L +++ mktemp ++ local LAST_ERR=/tmp/tmp.2r0qhy3JVa ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xMVZdE431L ++ cat /tmp/tmp.2r0qhy3JVa ++ rm /tmp/tmp.xMVZdE431L /tmp/tmp.2r0qhy3JVa ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z /rs0/myApp.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + key=/rs0/myApp.test.gz + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs0/myApp.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs0/myApp.test.gz found in oss-native' Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs0/myApp.test.gz found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_in_storage backup-oss-native oss rs1 myApp1.test.gz + local backup=backup-oss-native + local storage_type=oss + local replset=rs1 + local file=myApp1.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ local backup_name=backup-oss-native ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.IbsxXvJC1X ++ sed 's|oci://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.G3TbFQJ8Be ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IbsxXvJC1X ++ cat /tmp/tmp.G3TbFQJ8Be ++ rm /tmp/tmp.IbsxXvJC1X /tmp/tmp.G3TbFQJ8Be ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OAdzvYzsV3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZiIntKN74M +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.OAdzvYzsV3 +++ cat /tmp/tmp.ZiIntKN74M +++ rm /tmp/tmp.OAdzvYzsV3 /tmp/tmp.ZiIntKN74M +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ndo6hEkkW3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jlJLmM8YbV +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ndo6hEkkW3 +++ cat /tmp/tmp.jlJLmM8YbV +++ rm /tmp/tmp.ndo6hEkkW3 /tmp/tmp.jlJLmM8YbV +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f9ABH0CtIo +++ mktemp ++ local LAST_ERR=/tmp/tmp.WkELtzCzoS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.f9ABH0CtIo ++ cat /tmp/tmp.WkELtzCzoS ++ rm /tmp/tmp.f9ABH0CtIo /tmp/tmp.WkELtzCzoS ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ujcrF6H5qe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xaQD0cg0G3 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ujcrF6H5qe +++ cat /tmp/tmp.xaQD0cg0G3 +++ rm /tmp/tmp.ujcrF6H5qe /tmp/tmp.xaQD0cg0G3 +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eeSySfeefj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CjHje8qoN5 +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.eeSySfeefj +++ cat /tmp/tmp.CjHje8qoN5 +++ rm /tmp/tmp.eeSySfeefj /tmp/tmp.CjHje8qoN5 +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.poQMUdRIRD +++ mktemp ++ local LAST_ERR=/tmp/tmp.VoGIoEQlq2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.poQMUdRIRD ++ cat /tmp/tmp.VoGIoEQlq2 ++ rm /tmp/tmp.poQMUdRIRD /tmp/tmp.VoGIoEQlq2 ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z /rs1/myApp1.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + key=/rs1/myApp1.test.gz + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs1/myApp1.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs1/myApp1.test.gz found in oss-native' Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs1/myApp1.test.gz found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_in_storage backup-oss-native oss rs2 myApp2.test.gz + local backup=backup-oss-native + local storage_type=oss + local replset=rs2 + local file=myApp2.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-oss-native ++ local backup_name=backup-oss-native ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' +++ mktemp ++ sed 's|s3://||' ++ sed 's|oci://||' ++ sed 's|oss://||' ++ local LAST_OUT=/tmp/tmp.Ojpw8thDTb ++ sed 's|azure://||' ++ sed 's|gs://||' +++ mktemp ++ local LAST_ERR=/tmp/tmp.dOAY7uAxJm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ojpw8thDTb ++ cat /tmp/tmp.dOAY7uAxJm ++ rm /tmp/tmp.Ojpw8thDTb /tmp/tmp.dOAY7uAxJm ++ return 0 + backup_dest=operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + case ${storage_type} in ++ get_backup_storage_oss_field backup-oss-native endpointUrl ++ local backup_name=backup-oss-native ++ local field=endpointUrl ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3DFBtPdTZB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OXNw57DZQh +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.3DFBtPdTZB +++ cat /tmp/tmp.OXNw57DZQh +++ rm /tmp/tmp.3DFBtPdTZB /tmp/tmp.OXNw57DZQh +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.u3r8CyaIaU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gxkTJcNNsD +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.u3r8CyaIaU +++ cat /tmp/tmp.gxkTJcNNsD +++ rm /tmp/tmp.u3r8CyaIaU /tmp/tmp.gxkTJcNNsD +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json +++ mktemp ++ local LAST_OUT=/tmp/tmp.37JJlgCW6O ++ jq -r --arg storage oss-native --arg field endpointUrl '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_ERR=/tmp/tmp.olTYZbuXsd ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.37JJlgCW6O ++ cat /tmp/tmp.olTYZbuXsd ++ rm /tmp/tmp.37JJlgCW6O /tmp/tmp.olTYZbuXsd ++ return 0 + endpoint=https://oss-eu-central-1.aliyuncs.com ++ get_backup_storage_oss_field backup-oss-native region ++ local backup_name=backup-oss-native ++ local field=region ++ local cluster_name ++ local storage_name +++ get_backup_cluster_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.x4VuDmBefN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3s7Am1G9DC +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.clusterName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.clusterName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.x4VuDmBefN +++ cat /tmp/tmp.3s7Am1G9DC +++ rm /tmp/tmp.x4VuDmBefN /tmp/tmp.3s7Am1G9DC +++ return 0 ++ cluster_name=some-name +++ get_backup_storage_name backup-oss-native +++ local backup_name=backup-oss-native +++ kubectl_bin get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ej4URYaig2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.D7NBgYvr9f +++ local exit_status=0 +++ local timeout=4 +++ args=('get' 'psmdb-backup' 'backup-oss-native' '-o' 'jsonpath={.spec.storageName}') +++ local args +++ transformed_files=() +++ local transformed_files +++ arch_scheduling_enabled +++ [[ -n '' ]] +++ arch_scheduling_enabled +++ [[ -n '' ]] ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get psmdb-backup backup-oss-native -o 'jsonpath={.spec.storageName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Ej4URYaig2 +++ cat /tmp/tmp.D7NBgYvr9f +++ rm /tmp/tmp.Ej4URYaig2 /tmp/tmp.D7NBgYvr9f +++ return 0 ++ storage_name=oss-native ++ kubectl_bin get psmdb some-name -o json ++ jq -r --arg storage oss-native --arg field region '.spec.backup.storages[$storage].oss[$field]' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8nxEiwGNd3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bTBdVr6gGu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'json') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o json ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8nxEiwGNd3 ++ cat /tmp/tmp.bTBdVr6gGu ++ rm /tmp/tmp.8nxEiwGNd3 /tmp/tmp.bTBdVr6gGu ++ return 0 + region=eu-central-1 ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_existence_aws operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z /rs2/myApp2.test.gz https://s3.oss-eu-central-1.aliyuncs.com oss-native ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + key=/rs2/myApp2.test.gz + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + storage_name=oss-native + retry=0 + endpoint_arg=() + local endpoint_arg + [[ -n https://s3.oss-eu-central-1.aliyuncs.com ]] + endpoint_arg=(--endpoint-url "$endpoint") + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs2/myApp2.test.gz + echo 'Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs2/myApp2.test.gz found in oss-native' Backup operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z//rs2/myApp2.test.gz found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + insert_data_mongos 100501 myApp '' 27019 + local data=100501 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Dmtb65ygXW +++ mktemp ++ local LAST_ERR=/tmp/tmp.KTvnKLy9Jw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Dmtb65ygXW ++ cat /tmp/tmp.KTvnKLy9Jw ++ rm /tmp/tmp.Dmtb65ygXW /tmp/tmp.KTvnKLy9Jw ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.9KmvSibgez ++ mktemp + local LAST_ERR=/tmp/tmp.2vpArtuDBH + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9KmvSibgez Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("2f530d34-d9ff-4e4f-94d8-10229ad0cd48") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2vpArtuDBH + rm /tmp/tmp.9KmvSibgez /tmp/tmp.2vpArtuDBH + return 0 + insert_data_mongos 100501 myApp1 '' 27019 + local data=100501 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tcL2Xox139 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VCQwEXZHaY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tcL2Xox139 ++ cat /tmp/tmp.VCQwEXZHaY ++ rm /tmp/tmp.tcL2Xox139 /tmp/tmp.VCQwEXZHaY ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.dEkEXn8AE5 ++ mktemp + local LAST_ERR=/tmp/tmp.n5fsISvqrt + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dEkEXn8AE5 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("42a3dacc-4b3d-4e72-bc2e-dad1ec8458a8") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.n5fsISvqrt + rm /tmp/tmp.dEkEXn8AE5 /tmp/tmp.n5fsISvqrt + return 0 + insert_data_mongos 100501 myApp2 '' 27019 + local data=100501 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JgKLga0yik +++ mktemp ++ local LAST_ERR=/tmp/tmp.pka9s1teRW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JgKLga0yik ++ cat /tmp/tmp.pka9s1teRW ++ rm /tmp/tmp.JgKLga0yik /tmp/tmp.pka9s1teRW ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.bYp0JeHVRh ++ mktemp + local LAST_ERR=/tmp/tmp.2uqg3xMwFz + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bYp0JeHVRh Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("6de24f3e-eb39-4562-9c26-7e36cb11eb22") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2uqg3xMwFz + rm /tmp/tmp.bYp0JeHVRh /tmp/tmp.2uqg3xMwFz + return 0 + check_data -2nd + local postfix=-2nd ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:09:54+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4xFmKRELM6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.p5Ol61ciLD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4xFmKRELM6 ++ cat /tmp/tmp.p5Ol61ciLD ++ rm /tmp/tmp.4xFmKRELM6 /tmp/tmp.p5Ol61ciLD ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Y1vVlaCJCq ++ mktemp + local LAST_ERR=/tmp/tmp.zIvEHqi5ir + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Y1vVlaCJCq + cat /tmp/tmp.zIvEHqi5ir + rm /tmp/tmp.Y1vVlaCJCq /tmp/tmp.zIvEHqi5ir + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.hfrg7he8cN/find-2nd + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:09:56+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IImncCdz6z +++ mktemp ++ local LAST_ERR=/tmp/tmp.1mXE3yVKpt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IImncCdz6z ++ cat /tmp/tmp.1mXE3yVKpt ++ rm /tmp/tmp.IImncCdz6z /tmp/tmp.1mXE3yVKpt ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.527lph0aID ++ mktemp + local LAST_ERR=/tmp/tmp.zEBIZy62Ko + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.527lph0aID + cat /tmp/tmp.zEBIZy62Ko + rm /tmp/tmp.527lph0aID /tmp/tmp.zEBIZy62Ko + return 0 + compare_mongo_output find -2nd1 + local command=find + local postfix=-2nd1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.hfrg7he8cN/find-2nd1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:09:59+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U5CkL27alN +++ mktemp ++ local LAST_ERR=/tmp/tmp.sznmX5NgeI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U5CkL27alN ++ cat /tmp/tmp.sznmX5NgeI ++ rm /tmp/tmp.U5CkL27alN /tmp/tmp.sznmX5NgeI ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Fp67l7wlnd ++ mktemp + local LAST_ERR=/tmp/tmp.2dwzlCTynF + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Fp67l7wlnd + cat /tmp/tmp.2dwzlCTynF + rm /tmp/tmp.Fp67l7wlnd /tmp/tmp.2dwzlCTynF + return 0 + compare_mongo_output find -2nd2 + local command=find + local postfix=-2nd2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.hfrg7he8cN/find-2nd2 + return + run_restore backup-oss-native + local backup_name=backup-oss-native + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + log 'running restore restore-backup-oss-native' + set +o xtrace [2026-08-04T20:10:01+0000] running restore restore-backup-oss-native + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-oss-native/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-oss-native/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.319BsvXMsC ++ mktemp + local LAST_ERR=/tmp/tmp.ESfj7kywkE + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.319BsvXMsC perconaservermongodbrestore.psmdb.percona.com/restore-backup-oss-native created + cat /tmp/tmp.ESfj7kywkE + rm /tmp/tmp.319BsvXMsC /tmp/tmp.ESfj7kywkE + return 0 + wait_restore backup-oss-native some-name + local backup_name=backup-oss-native + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-oss-native object to be created.OK Waiting psmdb-restore/restore-backup-oss-native to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Uhb8Odv5jD +++ mktemp ++ local LAST_ERR=/tmp/tmp.VIqvtl1QYK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Uhb8Odv5jD ++ cat /tmp/tmp.VIqvtl1QYK ++ rm /tmp/tmp.Uhb8Odv5jD /tmp/tmp.VIqvtl1QYK ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6wMZoqdHwK +++ mktemp ++ local LAST_ERR=/tmp/tmp.UeIPcQ4zJZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6wMZoqdHwK ++ cat /tmp/tmp.UeIPcQ4zJZ ++ rm /tmp/tmp.6wMZoqdHwK /tmp/tmp.UeIPcQ4zJZ ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.g8O9eWb6vP +++ mktemp ++ local LAST_ERR=/tmp/tmp.7ETcINLdhk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.g8O9eWb6vP ++ cat /tmp/tmp.7ETcINLdhk ++ rm /tmp/tmp.g8O9eWb6vP /tmp/tmp.7ETcINLdhk ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1qqVgGtxZs +++ mktemp ++ local LAST_ERR=/tmp/tmp.0GgkLIaRY7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1qqVgGtxZs ++ cat /tmp/tmp.0GgkLIaRY7 ++ rm /tmp/tmp.1qqVgGtxZs /tmp/tmp.0GgkLIaRY7 ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.depv2tE9QH +++ mktemp ++ local LAST_ERR=/tmp/tmp.pywvHJDw6c ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.depv2tE9QH ++ cat /tmp/tmp.pywvHJDw6c ++ rm /tmp/tmp.depv2tE9QH /tmp/tmp.pywvHJDw6c ++ return 0 + [[ ready == ready ]] + echo .OK .OK + check_data + local postfix= ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:12:28+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dSeHreEj4K +++ mktemp ++ local LAST_ERR=/tmp/tmp.G396M3KUVr ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dSeHreEj4K ++ cat /tmp/tmp.G396M3KUVr ++ rm /tmp/tmp.dSeHreEj4K /tmp/tmp.G396M3KUVr ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.xiwbDIEmH6 ++ mktemp + local LAST_ERR=/tmp/tmp.UA4vI61fbY + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xiwbDIEmH6 + cat /tmp/tmp.UA4vI61fbY + rm /tmp/tmp.xiwbDIEmH6 /tmp/tmp.UA4vI61fbY + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:12:29+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IK34r5DliY +++ mktemp ++ local LAST_ERR=/tmp/tmp.U5yxWE0tYS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IK34r5DliY ++ cat /tmp/tmp.U5yxWE0tYS ++ rm /tmp/tmp.IK34r5DliY /tmp/tmp.U5yxWE0tYS ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.3HGeTAvM2g ++ mktemp + local LAST_ERR=/tmp/tmp.1SZBJFfb2V + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3HGeTAvM2g + cat /tmp/tmp.1SZBJFfb2V + rm /tmp/tmp.3HGeTAvM2g /tmp/tmp.1SZBJFfb2V + return 0 + compare_mongo_output find 1 + local command=find + local postfix=1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.hfrg7he8cN/find1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:12:32+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0SxkMd0pDV +++ mktemp ++ local LAST_ERR=/tmp/tmp.7lRlWMHNIq ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0SxkMd0pDV ++ cat /tmp/tmp.7lRlWMHNIq ++ rm /tmp/tmp.0SxkMd0pDV /tmp/tmp.7lRlWMHNIq ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.DweFjd7fnw ++ mktemp + local LAST_ERR=/tmp/tmp.bABT8Volgp + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DweFjd7fnw + cat /tmp/tmp.bABT8Volgp + rm /tmp/tmp.DweFjd7fnw /tmp/tmp.bABT8Volgp + return 0 + compare_mongo_output find 2 + local command=find + local postfix=2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.hfrg7he8cN/find2 + return + desc 'check backup and restore -- minio' + set +o xtrace ----------------------------------------------------------------------------------- check backup and restore -- minio ----------------------------------------------------------------------------------- + check_backup_in_storage backup-minio minio rs0 myApp.test.gz + local backup=backup-minio + local storage_type=minio + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest backup-minio ++ local backup_name=backup-minio ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ sed 's|s3://||' ++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|gs://||' ++ sed 's|azure://||' ++ sed 's|oss://||' ++ sed 's|oci://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dGtkMHVFi4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FcsrLV1VjM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' 'backup-minio' '-o' 'jsonpath={.status.destination}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dGtkMHVFi4 ++ cat /tmp/tmp.FcsrLV1VjM ++ rm /tmp/tmp.dGtkMHVFi4 /tmp/tmp.FcsrLV1VjM ++ return 0 + backup_dest=operator-testing/2026-08-04T19:54:40Z + case ${storage_type} in + endpoint=minio-service + aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:54:40Z/rs0/myApp.test.gz + local endpoint=http://minio-service:9000 + shift + grep myApp.test.gz + kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:54:40Z/rs0/myApp.test.gz ++ mktemp + local LAST_OUT=/tmp/tmp.KFUHlg3Kgk ++ mktemp + local LAST_ERR=/tmp/tmp.HG52QjeGtW + local exit_status=0 + local timeout=4 + args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/2026-08-04T19:54:40Z/rs0/myApp.test.gz') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2026-08-04T19:54:40Z/rs0/myApp.test.gz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KFUHlg3Kgk + cat /tmp/tmp.HG52QjeGtW All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.KFUHlg3Kgk /tmp/tmp.HG52QjeGtW + return 0 2026-08-04 19:54:45 55 myApp.test.gz + insert_data_mongos 100501 myApp '' 27019 + local data=100501 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.brnesMGUS8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JbtzIgZZg6 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.brnesMGUS8 ++ cat /tmp/tmp.JbtzIgZZg6 ++ rm /tmp/tmp.brnesMGUS8 /tmp/tmp.JbtzIgZZg6 ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.kbNArmKbiu ++ mktemp + local LAST_ERR=/tmp/tmp.rs9uPW3Enl + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kbNArmKbiu Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("256e0e0b-35b8-44a2-a1e1-72672b817c8c") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.rs9uPW3Enl + rm /tmp/tmp.kbNArmKbiu /tmp/tmp.rs9uPW3Enl + return 0 + insert_data_mongos 100501 myApp1 '' 27019 + local data=100501 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rxagt5LKtm +++ mktemp ++ local LAST_ERR=/tmp/tmp.pJzRw5nQAB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rxagt5LKtm ++ cat /tmp/tmp.pJzRw5nQAB ++ rm /tmp/tmp.rxagt5LKtm /tmp/tmp.pJzRw5nQAB ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.cqRUzvaiU0 ++ mktemp + local LAST_ERR=/tmp/tmp.cIzqleJhrz + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cqRUzvaiU0 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("5193e2fd-f67c-4af9-b5a4-47b85d3a3fe1") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.cIzqleJhrz + rm /tmp/tmp.cqRUzvaiU0 /tmp/tmp.cIzqleJhrz + return 0 + insert_data_mongos 100501 myApp2 '' 27019 + local data=100501 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ddkDw5FCiw +++ mktemp ++ local LAST_ERR=/tmp/tmp.BGNI3AITvY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ddkDw5FCiw ++ cat /tmp/tmp.BGNI3AITvY ++ rm /tmp/tmp.ddkDw5FCiw /tmp/tmp.BGNI3AITvY ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.f4QXep4bCG ++ mktemp + local LAST_ERR=/tmp/tmp.lc4vyF3GcG + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f4QXep4bCG Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8c61af5b-876b-4caf-985d-44787bc03a76") } Percona Server for MongoDB server version: v8.0.26-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.lc4vyF3GcG + rm /tmp/tmp.f4QXep4bCG /tmp/tmp.lc4vyF3GcG + return 0 + check_data -2nd + local postfix=-2nd ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:13:00+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9aM1Ysal3m +++ mktemp ++ local LAST_ERR=/tmp/tmp.ketn8IS2WM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9aM1Ysal3m ++ cat /tmp/tmp.ketn8IS2WM ++ rm /tmp/tmp.9aM1Ysal3m /tmp/tmp.ketn8IS2WM ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.MaXZaEd4gT ++ mktemp + local LAST_ERR=/tmp/tmp.rK9L2kFoOG + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MaXZaEd4gT + cat /tmp/tmp.rK9L2kFoOG + rm /tmp/tmp.MaXZaEd4gT /tmp/tmp.rK9L2kFoOG + return 0 + compare_mongo_output find -2nd + local command=find + local postfix=-2nd + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.hfrg7he8cN/find-2nd + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:13:03+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.15AWsIae0I +++ mktemp ++ local LAST_ERR=/tmp/tmp.0pLyWBeGpB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.15AWsIae0I ++ cat /tmp/tmp.0pLyWBeGpB ++ rm /tmp/tmp.15AWsIae0I /tmp/tmp.0pLyWBeGpB ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ieQwMf80Aa ++ mktemp + local LAST_ERR=/tmp/tmp.OVTKPAPaqJ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ieQwMf80Aa + cat /tmp/tmp.OVTKPAPaqJ + rm /tmp/tmp.ieQwMf80Aa /tmp/tmp.OVTKPAPaqJ + return 0 + compare_mongo_output find -2nd1 + local command=find + local postfix=-2nd1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.hfrg7he8cN/find-2nd1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 -2nd2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=-2nd2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:13:05+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp2\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CBg3ULUlsO +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z0gqO4QqDF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CBg3ULUlsO ++ cat /tmp/tmp.Z0gqO4QqDF ++ rm /tmp/tmp.CBg3ULUlsO /tmp/tmp.Z0gqO4QqDF ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Pzvc9y9fUU ++ mktemp + local LAST_ERR=/tmp/tmp.t7FqTvR2D5 + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Pzvc9y9fUU + cat /tmp/tmp.t7FqTvR2D5 + rm /tmp/tmp.Pzvc9y9fUU /tmp/tmp.t7FqTvR2D5 + return 0 + compare_mongo_output find -2nd2 + local command=find + local postfix=-2nd2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.hfrg7he8cN/find-2nd2 + return + run_restore backup-minio + local backup_name=backup-minio + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + log 'running restore restore-backup-minio' + set +o xtrace [2026-08-04T20:13:07+0000] running restore restore-backup-minio + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.jhvkR1JIyY ++ mktemp + local LAST_ERR=/tmp/tmp.r264IRvvSv + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jhvkR1JIyY perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created + cat /tmp/tmp.r264IRvvSv + rm /tmp/tmp.jhvkR1JIyY /tmp/tmp.r264IRvvSv + return 0 + wait_restore backup-minio some-name + local backup_name=backup-minio + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio object to be created.OK Waiting psmdb-restore/restore-backup-minio to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LLZDOHFbiS +++ mktemp ++ local LAST_ERR=/tmp/tmp.WisE8KNOPl ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LLZDOHFbiS ++ cat /tmp/tmp.WisE8KNOPl ++ rm /tmp/tmp.LLZDOHFbiS /tmp/tmp.WisE8KNOPl ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8c3P7ihQDO +++ mktemp ++ local LAST_ERR=/tmp/tmp.qjjJgsrgP9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8c3P7ihQDO ++ cat /tmp/tmp.qjjJgsrgP9 ++ rm /tmp/tmp.8c3P7ihQDO /tmp/tmp.qjjJgsrgP9 ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UyJD3rTwYk +++ mktemp ++ local LAST_ERR=/tmp/tmp.0JfcyavoPu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UyJD3rTwYk ++ cat /tmp/tmp.0JfcyavoPu ++ rm /tmp/tmp.UyJD3rTwYk /tmp/tmp.0JfcyavoPu ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tra3Pf71Q6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gHLONPxZwa ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Tra3Pf71Q6 ++ cat /tmp/tmp.gHLONPxZwa ++ rm /tmp/tmp.Tra3Pf71Q6 /tmp/tmp.gHLONPxZwa ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yjCG4aXB0Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.vQrvHgdQkt ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yjCG4aXB0Q ++ cat /tmp/tmp.vQrvHgdQkt ++ rm /tmp/tmp.yjCG4aXB0Q /tmp/tmp.vQrvHgdQkt ++ return 0 + [[ ready == ready ]] + echo .OK .OK + check_data + local postfix= ++ seq 0 2 + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 '' .svc.cluster.local myApp test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix= + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp' + set +o xtrace [2026-08-04T20:15:22+0000] running db.test.command() in myApp ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zFKmkIeHCr +++ mktemp ++ local LAST_ERR=/tmp/tmp.i0E3g80xfu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zFKmkIeHCr ++ cat /tmp/tmp.i0E3g80xfu ++ rm /tmp/tmp.zFKmkIeHCr /tmp/tmp.i0E3g80xfu ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.WW5iMrnPn3 ++ mktemp + local LAST_ERR=/tmp/tmp.cQVo1irxgW + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WW5iMrnPn3 + cat /tmp/tmp.cQVo1irxgW + rm /tmp/tmp.WW5iMrnPn3 /tmp/tmp.cQVo1irxgW + return 0 + compare_mongo_output find '' + local command=find + local postfix= + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.hfrg7he8cN/find + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 1 .svc.cluster.local myApp1 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=1 + local suffix=.svc.cluster.local + local database=myApp1 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp1' + set +o xtrace [2026-08-04T20:15:24+0000] running db.test.command() in myApp1 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp1\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nKLg7fKNul +++ mktemp ++ local LAST_ERR=/tmp/tmp.5dbOoAFQUC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nKLg7fKNul ++ cat /tmp/tmp.5dbOoAFQUC ++ rm /tmp/tmp.nKLg7fKNul /tmp/tmp.5dbOoAFQUC ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.AjB93JBcMr ++ mktemp + local LAST_ERR=/tmp/tmp.aMbgNJ14SJ + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AjB93JBcMr + cat /tmp/tmp.aMbgNJ14SJ + rm /tmp/tmp.AjB93JBcMr /tmp/tmp.aMbgNJ14SJ + return 0 + compare_mongo_output find 1 + local command=find + local postfix=1 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.hfrg7he8cN/find1 + return + for i in $(seq 0 2) + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-12795 2 .svc.cluster.local myApp2 test 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local postfix=2 + local suffix=.svc.cluster.local + local database=myApp2 + local collection=test + local port=27019 + local tls=false + local retries=3 + local retry_delay=10 + [[ false == true ]] + mongos_command=run_mongos + log 'running db.test.command() in myApp2' + set +o xtrace [2026-08-04T20:15:26+0000] running db.test.command() in myApp2 ++ seq 1 3 + for attempt in $(seq 1 "$retries") + run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-12795 mongodb .svc.cluster.local '' 27019 + local 'command=use myApp2\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-12795 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kJzUAqWuYO +++ mktemp ++ local LAST_ERR=/tmp/tmp.xUsL5HbEyv ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=psmdb-client' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kJzUAqWuYO ++ cat /tmp/tmp.xUsL5HbEyv ++ rm /tmp/tmp.kJzUAqWuYO /tmp/tmp.xUsL5HbEyv ++ return 0 + local client_container=psmdb-client-757cd65695-qtfvx + kubectl_bin exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Xm5fmLocus ++ mktemp + local LAST_ERR=/tmp/tmp.0VOxX6sOsB + local exit_status=0 + local timeout=4 + args=('exec' 'psmdb-client-757cd65695-qtfvx' '--' 'bash' '-c' 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-757cd65695-qtfvx -- bash -c 'export HOME=/tmp; printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-12795.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Xm5fmLocus + cat /tmp/tmp.0VOxX6sOsB + rm /tmp/tmp.Xm5fmLocus /tmp/tmp.0VOxX6sOsB + return 0 + compare_mongo_output find 2 + local command=find + local postfix=2 + [[ 0 -ne 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.hfrg7he8cN/find2 + return + desc 'delete backup and check if it is removed from bucket -- minio' + set +o xtrace ----------------------------------------------------------------------------------- delete backup and check if it is removed from bucket -- minio ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb-backup --all ++ mktemp + local LAST_OUT=/tmp/tmp.7PTsVaSbt5 ++ mktemp + local LAST_ERR=/tmp/tmp.iX1S8PSwqO + local exit_status=0 + local timeout=4 + args=('delete' 'psmdb-backup' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb-backup --all + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7PTsVaSbt5 perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-sharded-12795 namespace perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-sharded-12795 namespace perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted from demand-backup-sharded-12795 namespace perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-sharded-12795 namespace perconaservermongodbbackup.psmdb.percona.com "backup-oss-native" deleted from demand-backup-sharded-12795 namespace + cat /tmp/tmp.iX1S8PSwqO + rm /tmp/tmp.7PTsVaSbt5 /tmp/tmp.iX1S8PSwqO + return 0 ++ grep -c _ ++ aws_cli_minio http://minio-service:9000 s3 ls s3://operator-testing/ ++ local endpoint=http://minio-service:9000 ++ shift ++ kubectl_bin run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ cat +++ mktemp ++ local LAST_OUT=/tmp/tmp.N4AKDosByZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.KJrhq5qmiI ++ local exit_status=0 ++ local timeout=4 ++ args=('run' '-i' '--rm' 'aws-cli' '--image=docker.io/amazon/aws-cli:2.34.60' '--restart=Never' '--env=AWS_ACCESS_KEY_ID=some-access-key' '--env=AWS_SECRET_ACCESS_KEY=some-secret-key' '--env=AWS_DEFAULT_REGION=us-east-1' '--' '--no-verify-ssl' '--endpoint-url' 'http://minio-service:9000' 's3' 'ls' 's3://operator-testing/') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl run -i --rm aws-cli --image=docker.io/amazon/aws-cli:2.34.60 --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 -- --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.N4AKDosByZ ++ cat /tmp/tmp.KJrhq5qmiI All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. ++ rm /tmp/tmp.N4AKDosByZ /tmp/tmp.KJrhq5qmiI ++ return 0 + backup_exists=0 + [[ 0 -eq 1 ]] + '[' -z '' ']' + check_backup_deletion_aws operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:22Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded/2026-08-04T19:56:22Z + key=/rs0/myApp.test.gz + storage_name=aws-s3 + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_aws_credentials + local secret_name=aws-s3-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + echo 'AWS credentials already set in environment' AWS credentials already set in environment + return 0 + aws s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3' Backup psmdb-demand-backup-sharded/2026-08-04T19:56:22Z/rs0/myApp.test.gz in bucket operator-testing not found in aws-s3 + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + check_backup_deletion_gcs operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z /rs0/myApp.test.gz + backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z + obj=/rs0/myApp.test.gz + storage_name=gcp-cs + retry=0 + gcs_path=gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz + gsutil ls gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz + echo 'Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz not found in gcp-cs' Backup gs://operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:56:56Z/rs0/myApp.test.gz not found in gcp-cs + check_backup_deletion_azure operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z /rs0/myApp.test.gz ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ sed 's|https://[^/]*\.blob\.core\.windows\.net/||' + url_path=operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f1 + container=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded/2026-08-04T19:55:13Z ++ cut -d/ -f2- + blob_prefix=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z + blob=/rs0/myApp.test.gz + storage_name=azure-blob + retry=0 + blob_path=psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz + az storage blob show --container-name operator-testing --name psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob' Backup psmdb-demand-backup-sharded/2026-08-04T19:55:13Z/rs0/myApp.test.gz in container operator-testing not found in azure-blob + check_backup_deletion_oss operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z /rs0/myApp.test.gz https://oss-eu-central-1.aliyuncs.com eu-central-1 ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f1 + bucket=operator-testing ++ echo operator-testing/psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z ++ cut -d/ -f2- + key_prefix=psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z + key=/rs0/myApp.test.gz ++ echo https://oss-eu-central-1.aliyuncs.com ++ /usr/sbin/sed 's|://oss-|://s3.oss-|' + endpoint=https://s3.oss-eu-central-1.aliyuncs.com + region=eu-central-1 + storage_name=oss-native + retry=0 + _aws_env_save + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + setup_alibaba_cloud_s3_credentials eu-central-1 + local region=eu-central-1 + local secret_name=alibaba-cloud-secret + echo 'Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret' Setting up Alibaba Cloud S3-compatible credentials from secret: alibaba-cloud-secret + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + aws --endpoint-url https://s3.oss-eu-central-1.aliyuncs.com s3api head-object --bucket operator-testing --key psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs0/myApp.test.gz + echo 'Backup psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native' Backup psmdb-demand-backup-sharded-oss-native/2026-08-04T19:55:47Z/rs0/myApp.test.gz in bucket operator-testing not found in oss-native + _aws_env_restore + local trace_was_on=0 + [[ ehxB == *x* ]] + trace_was_on=1 + set +x + return 0 + destroy demand-backup-sharded-12795 + local namespace=demand-backup-sharded-12795 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_backups + desc 'Delete psmdb-backup' + set +o xtrace ----------------------------------------------------------------------------------- Delete psmdb-backup ----------------------------------------------------------------------------------- ++ kubectl_bin get psmdb-backup --no-headers ++ wc -l +++ mktemp ++ local LAST_OUT=/tmp/tmp.yHGfbEf0ft +++ mktemp ++ local LAST_ERR=/tmp/tmp.ggl7Kwgtol ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb-backup' '--no-headers') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup --no-headers ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yHGfbEf0ft ++ cat /tmp/tmp.ggl7Kwgtol No resources found in demand-backup-sharded-12795 namespace. ++ rm /tmp/tmp.yHGfbEf0ft /tmp/tmp.ggl7Kwgtol ++ return 0 + '[' 0 '!=' 0 ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.nnvytPOX3r ++ mktemp + local LAST_ERR=/tmp/tmp.e02OLygLLs + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nnvytPOX3r customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.e02OLygLLs + rm /tmp/tmp.nnvytPOX3r /tmp/tmp.e02OLygLLs + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.l8E7mJLNlj ++ mktemp + local LAST_ERR=/tmp/tmp.4afFw0iTHZ + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l8E7mJLNlj + cat /tmp/tmp.4afFw0iTHZ + rm /tmp/tmp.l8E7mJLNlj /tmp/tmp.4afFw0iTHZ + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.zehhqoCSJh ++ mktemp + local LAST_ERR=/tmp/tmp.QpnDf4lu47 + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zehhqoCSJh + cat /tmp/tmp.QpnDf4lu47 + rm /tmp/tmp.zehhqoCSJh /tmp/tmp.QpnDf4lu47 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.OPxEuNyEvc ++ mktemp + local LAST_ERR=/tmp/tmp.0VxGLVth8c + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OPxEuNyEvc + cat /tmp/tmp.0VxGLVth8c + rm /tmp/tmp.OPxEuNyEvc /tmp/tmp.0VxGLVth8c + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.W9U8rEvxUu ++ mktemp + local LAST_ERR=/tmp/tmp.goplYcycNo + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.W9U8rEvxUu + cat /tmp/tmp.goplYcycNo + rm /tmp/tmp.W9U8rEvxUu /tmp/tmp.goplYcycNo + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.46mTnfKEKV ++ mktemp + local LAST_ERR=/tmp/tmp.ERC6h6Iire + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.46mTnfKEKV clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.ERC6h6Iire + rm /tmp/tmp.46mTnfKEKV /tmp/tmp.ERC6h6Iire + return 0 + destroy_cert_manager + [[ gke == rancher ]] + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.8isDKhV6fM ++ mktemp + local LAST_ERR=/tmp/tmp.mo03Gh5g8u + local exit_status=0 + local timeout=4 + args=('delete' '-f' 'https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.8isDKhV6fM + cat /tmp/tmp.mo03Gh5g8u Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.8isDKhV6fM + cat /tmp/tmp.mo03Gh5g8u Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.8isDKhV6fM + cat /tmp/tmp.mo03Gh5g8u Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.8isDKhV6fM + cat /tmp/tmp.mo03Gh5g8u Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.21.0/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.8isDKhV6fM /tmp/tmp.mo03Gh5g8u + return 1 + true + [[ -n '' ]] + '[' -n psmdb-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-sharded-12795 + rm -rf /tmp/tmp.hfrg7he8cN + kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator ++ mktemp ++ mktemp + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.NxC5ykh1jZ + local LAST_OUT=/tmp/tmp.OwtHoOtfag ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.upYMtIODYb + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'demand-backup-sharded-12795') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.cIkK1TBxHK + local exit_status=0 + local timeout=4 + args=('delete' '--grace-period=0' '--force=true' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace demand-backup-sharded-12795 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace psmdb-operator ==================================================================================================== [PASSED] e2e-tests/test_pytest_wrapper.py::test_bash_wrapper ==================================================================================================== ----------------------------------------- generated xml file: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/reports/demand-backup-sharded.xml ------------------------------------------ ------------------------------------ Generated html report: file:///mnt/jenkins/workspace/cloud-psmdb-operator_PR-2491/e2e-tests/reports/demand-backup-sharded.html ------------------------------------ ==================================================================================== 1 passed in 2021.73s (0:33:41) ==================================================================================== + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OwtHoOtfag namespace "psmdb-operator" force deleted + cat /tmp/tmp.cIkK1TBxHK Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.OwtHoOtfag /tmp/tmp.cIkK1TBxHK + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NxC5ykh1jZ namespace "demand-backup-sharded-12795" force deleted + cat /tmp/tmp.upYMtIODYb Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.NxC5ykh1jZ /tmp/tmp.upYMtIODYb + return 0