============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-7.0.8-100.fc43.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.1.1', 'pluggy': '1.6.0'}, 'Plugins': {'html': '4.2.0', 'metadata': '3.1.1', 'dependency': '0.6.1', 'json-report': '1.5.0'}, 'CI': 'true', 'BUILD_NUMBER': '22', 'BUILD_ID': '22', 'BUILD_URL': 'https://cloud.cd.percona.com/job/cloud-psmdb-operator/job/PR-2058/22/', 'NODE_NAME': 'hcloud-3hmxjv56rzynxord', 'JOB_NAME': 'cloud-psmdb-operator/PR-2058', 'BUILD_TAG': 'jenkins-cloud-psmdb-operator-PR-2058-22', 'EXECUTOR_NUMBER': '0', 'JENKINS_URL': 'https://cloud.cd.percona.com/', 'WORKSPACE': '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058', 'GIT_COMMIT': '78121c56762758785b75e9b7c017e15fc887ae5d', 'GIT_URL': 'https://github.com/percona/percona-server-mongodb-operator.git', 'GIT_BRANCH': 'PR-2058'} rootdir: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058 configfile: pyproject.toml plugins: html-4.2.0, metadata-3.1.1, dependency-0.6.1, json-report-1.5.0 collecting ... collected 1 item e2e-tests/test_pytest_wrapper.py::test_bash_wrapper[upgrade-consistency-sharded-tls] [02:00:41.087948] WARNING kubectl error: Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 [02:00:43.549557] INFO Environment variables: KUBE_VERSION=1.33 EKS=0 GKE=1 OPENSHIFT= MINIKUBE= API=psmdb.percona.com/v1 REGISTRY_NAME=docker.io GIT_COMMIT=78121c56762758785b75e9b7c017e15fc887ae5d GIT_BRANCH=PR-2058 OPERATOR_VERSION=1.23.0 IMAGE=docker.io/perconalab/percona-server-mongodb-ope rator:PR-2058-78121c567 IMAGE_MONGOD=docker.io/perconalab/percona-server-mong odb-operator:main-mongod8.0 IMAGE_MONGOD_CHAIN=docker.io/perconalab/percona-serve r-mongodb-operator:main-mongod6.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod7.0 docker.io/perconalab/percona-server-mongodb-operator: main-mongod8.0 IMAGE_SEARCH=perconalab/percona-server-mongodb-operat or:main-mongot IMAGE_BACKUP=docker.io/perconalab/percona-server-mong odb-operator:main-backup IMAGE_PMM_CLIENT=docker.io/percona/pmm-client:2.44.1- 1 IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:dev- latest IMAGE_PMM3_CLIENT=docker.io/perconalab/pmm-client:3-d ev-latest IMAGE_PMM3_SERVER=docker.io/perconalab/pmm-server:3-d ev-latest IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:mai n-logcollector IMAGE_CLUSTERSYNC=docker.io/percona/percona-clustersy nc-mongodb:0.9.0 CERT_MANAGER_VER=1.19.3 CHAOS_MESH_VER=2.7.1 MINIO_VER=5.4.0 PMM_SERVER_VER=9.9.9 CLEAN_NAMESPACE=1 DELETE_CRD_ON_START=1 SKIP_DELETE=0 SKIP_BACKUPS_TO_AWS_GCP_AZURE= UPDATE_COMPARE_FILES=0 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Debug logging disabled. + CLUSTER=some-name + main + create_infra upgrade-consistency-sharded-tls-2846 + local ns=upgrade-consistency-sharded-tls-2846 + echo upgrade-consistency-sharded-tls-2846 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.wClze6BWSv ++ mktemp + local LAST_ERR=/tmp/tmp.5MQBgPp3r1 + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml' '--ignore-not-found' '--wait=false') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wClze6BWSv + cat /tmp/tmp.5MQBgPp3r1 + rm /tmp/tmp.wClze6BWSv /tmp/tmp.5MQBgPp3r1 + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.AcrocCJxt2 ++ mktemp + local LAST_ERR=/tmp/tmp.YoMM28LFtd + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AcrocCJxt2 + cat /tmp/tmp.YoMM28LFtd + rm /tmp/tmp.AcrocCJxt2 /tmp/tmp.YoMM28LFtd + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbclustersyncs" + : + kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.9l6hhJhIie ++ mktemp + local LAST_ERR=/tmp/tmp.AEAPe1z0wl + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9l6hhJhIie + cat /tmp/tmp.AEAPe1z0wl + rm /tmp/tmp.9l6hhJhIie /tmp/tmp.AEAPe1z0wl + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.sEhqelqvNh ++ mktemp + local LAST_ERR=/tmp/tmp.v4Cvu1Xmet + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sEhqelqvNh + cat /tmp/tmp.v4Cvu1Xmet + rm /tmp/tmp.sEhqelqvNh /tmp/tmp.v4Cvu1Xmet + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.HIw2fsopAZ ++ mktemp + local LAST_ERR=/tmp/tmp.bI7HPC9y2a + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HIw2fsopAZ + cat /tmp/tmp.bI7HPC9y2a + rm /tmp/tmp.HIw2fsopAZ /tmp/tmp.bI7HPC9y2a + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.qsdfsYoXrN ++ mktemp + local LAST_ERR=/tmp/tmp.9FdcxdjD1m + local exit_status=0 + local timeout=4 + args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qsdfsYoXrN + cat /tmp/tmp.9FdcxdjD1m + rm /tmp/tmp.qsdfsYoXrN /tmp/tmp.9FdcxdjD1m + return 0 + check_crd_for_deletion PR-2058-78121c567 + local git_tag=PR-2058-78121c567 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2058-78121c567/deploy/crd.yaml ++ /usr/sbin/sed s/---//g ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HGL7lcLNhb +++ mktemp ++ local LAST_ERR=/tmp/tmp.ff5ZyIafxW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.HGL7lcLNhb ++ cat /tmp/tmp.ff5ZyIafxW Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.HGL7lcLNhb ++ cat /tmp/tmp.ff5ZyIafxW Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.HGL7lcLNhb ++ cat /tmp/tmp.ff5ZyIafxW Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.HGL7lcLNhb ++ cat /tmp/tmp.ff5ZyIafxW Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.HGL7lcLNhb /tmp/tmp.ff5ZyIafxW ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + [[ -n '' ]] + awk '{print$1}' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace + xargs kubectl delete ns ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.yoeGM9QNi4 + local LAST_OUT=/tmp/tmp.LSJ6UQRuqO ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.QW2or0K5SU + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + local LAST_ERR=/tmp/tmp.3G2bI0r66S + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yoeGM9QNi4 + cat /tmp/tmp.QW2or0K5SU + rm /tmp/tmp.yoeGM9QNi4 /tmp/tmp.QW2or0K5SU + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LSJ6UQRuqO + cat /tmp/tmp.3G2bI0r66S + rm /tmp/tmp.LSJ6UQRuqO /tmp/tmp.3G2bI0r66S + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.AcJ9iMwVEL ++ mktemp + local LAST_ERR=/tmp/tmp.NC4BTs4ywv + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AcJ9iMwVEL + cat /tmp/tmp.NC4BTs4ywv + rm /tmp/tmp.AcJ9iMwVEL /tmp/tmp.NC4BTs4ywv + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.8qagdWyDtP ++ mktemp + local LAST_ERR=/tmp/tmp.4YWhC7N57D + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8qagdWyDtP namespace/psmdb-operator created + cat /tmp/tmp.4YWhC7N57D + rm /tmp/tmp.8qagdWyDtP /tmp/tmp.4YWhC7N57D + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.OPPtDEqMe2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q2Mau0XAn6 ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OPPtDEqMe2 ++ cat /tmp/tmp.Q2Mau0XAn6 ++ rm /tmp/tmp.OPPtDEqMe2 /tmp/tmp.Q2Mau0XAn6 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.AN10MeOp67 ++ mktemp + local LAST_ERR=/tmp/tmp.LviAIMTagG + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15' '--namespace=psmdb-operator') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AN10MeOp67 Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15" modified. + cat /tmp/tmp.LviAIMTagG + rm /tmp/tmp.AN10MeOp67 /tmp/tmp.LviAIMTagG + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.BH8IXaypL7 ++ mktemp + local LAST_ERR=/tmp/tmp.PUHFtH4ysy + local exit_status=0 + local timeout=4 + args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BH8IXaypL7 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.PUHFtH4ysy + rm /tmp/tmp.BH8IXaypL7 /tmp/tmp.PUHFtH4ysy + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.lD8tg82O2j ++ mktemp + local LAST_ERR=/tmp/tmp.hqMXqv8kqe + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lD8tg82O2j clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.hqMXqv8kqe + rm /tmp/tmp.lD8tg82O2j /tmp/tmp.hqMXqv8kqe + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + [[ -n '' ]] + cat - + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.D1sewOH8ei ++ mktemp + local LAST_ERR=/tmp/tmp.NOeOFZUlYt + local exit_status=0 + local timeout=4 + args=('apply' '-n' 'psmdb-operator' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D1sewOH8ei deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.NOeOFZUlYt + rm /tmp/tmp.D1sewOH8ei /tmp/tmp.NOeOFZUlYt + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.LMhYnIWpQ4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5xf3o5Xj8j ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LMhYnIWpQ4 ++ cat /tmp/tmp.5xf3o5Xj8j ++ rm /tmp/tmp.LMhYnIWpQ4 /tmp/tmp.5xf3o5Xj8j ++ return 0 + wait_operator_pod percona-server-mongodb-operator-66cd7b75cf-rdgm5 + local pod=percona-server-mongodb-operator-66cd7b75cf-rdgm5 + set +o xtrace waiting for pod/percona-server-mongodb-operator-66cd7b75cf-rdgm5 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.bvxjtiGqxj +++ mktemp ++ local LAST_ERR=/tmp/tmp.EccxjxTt7J ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bvxjtiGqxj ++ cat /tmp/tmp.EccxjxTt7J ++ rm /tmp/tmp.bvxjtiGqxj /tmp/tmp.EccxjxTt7J ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-rdgm5 ++ mktemp + local LAST_OUT=/tmp/tmp.c0OmVTf01X ++ mktemp + local LAST_ERR=/tmp/tmp.RAdtw8Lkjy + local exit_status=0 + local timeout=4 + args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-66cd7b75cf-rdgm5') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-66cd7b75cf-rdgm5 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.c0OmVTf01X + cat /tmp/tmp.RAdtw8Lkjy + rm /tmp/tmp.c0OmVTf01X /tmp/tmp.RAdtw8Lkjy + return 0 2026-07-17T02:01:43.725Z INFO setup Manager starting up {"gitCommit": "78121c56762758785b75e9b7c017e15fc887ae5d", "gitBranch": "PR-2058-78121c567", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"} + create_namespace upgrade-consistency-sharded-tls-2846 + local namespace=upgrade-consistency-sharded-tls-2846 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + [[ -n '' ]] + desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-2846' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces upgrade-consistency-sharded-tls-2846 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace upgrade-consistency-sharded-tls-2846 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.TMmbpsE6rz ++ mktemp + local LAST_OUT=/tmp/tmp.cyhOflgIoI + local LAST_ERR=/tmp/tmp.uOsaUblcmR + local exit_status=0 + local timeout=4 + args=('get' 'ns') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.mNRdKtahNC + local exit_status=0 + local timeout=4 + args=('delete' 'namespace' 'upgrade-consistency-sharded-tls-2846' '--ignore-not-found') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace upgrade-consistency-sharded-tls-2846 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TMmbpsE6rz + cat /tmp/tmp.uOsaUblcmR + rm /tmp/tmp.TMmbpsE6rz /tmp/tmp.uOsaUblcmR + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cyhOflgIoI + cat /tmp/tmp.mNRdKtahNC + rm /tmp/tmp.cyhOflgIoI /tmp/tmp.mNRdKtahNC + return 0 + kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-2846 ++ mktemp + local LAST_OUT=/tmp/tmp.1aIzHvvmdX ++ mktemp + local LAST_ERR=/tmp/tmp.5NZuk385Cc + local exit_status=0 + local timeout=4 + args=('wait' '--for=delete' 'namespace' 'upgrade-consistency-sharded-tls-2846') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-2846 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1aIzHvvmdX + cat /tmp/tmp.5NZuk385Cc + rm /tmp/tmp.1aIzHvvmdX /tmp/tmp.5NZuk385Cc + return 0 + desc 'create namespace upgrade-consistency-sharded-tls-2846' + set +o xtrace ----------------------------------------------------------------------------------- create namespace upgrade-consistency-sharded-tls-2846 ----------------------------------------------------------------------------------- + kubectl_bin create namespace upgrade-consistency-sharded-tls-2846 ++ mktemp + local LAST_OUT=/tmp/tmp.pQNWkW7iRF ++ mktemp + local LAST_ERR=/tmp/tmp.VviQYylFPg + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'upgrade-consistency-sharded-tls-2846') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace upgrade-consistency-sharded-tls-2846 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pQNWkW7iRF namespace/upgrade-consistency-sharded-tls-2846 created + cat /tmp/tmp.VviQYylFPg + rm /tmp/tmp.pQNWkW7iRF /tmp/tmp.VviQYylFPg + return 0 + set_kube_ctx upgrade-consistency-sharded-tls-2846 + local namespace=upgrade-consistency-sharded-tls-2846 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.d9NXP8Xoql +++ mktemp ++ local LAST_ERR=/tmp/tmp.EkENyuSkcX ++ local exit_status=0 ++ local timeout=4 ++ args=('config' 'current-context') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d9NXP8Xoql ++ cat /tmp/tmp.EkENyuSkcX ++ rm /tmp/tmp.d9NXP8Xoql /tmp/tmp.EkENyuSkcX ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15 --namespace=upgrade-consistency-sharded-tls-2846 ++ mktemp + local LAST_OUT=/tmp/tmp.mgcHdU9QiC ++ mktemp + local LAST_ERR=/tmp/tmp.C08QtsKIlQ + local exit_status=0 + local timeout=4 + args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15' '--namespace=upgrade-consistency-sharded-tls-2846') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15 --namespace=upgrade-consistency-sharded-tls-2846 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mgcHdU9QiC Context "gke_cloud-dev-112233_us-central1_jen-psmdb-2058-78121c567-22-cluster15" modified. + cat /tmp/tmp.C08QtsKIlQ + rm /tmp/tmp.mgcHdU9QiC /tmp/tmp.C08QtsKIlQ + return 0 + deploy_cert_manager + [[ gke == rancher ]] + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin -n cert-manager delete clusterissuer --all ++ mktemp + local LAST_OUT=/tmp/tmp.qiUHlTJvOe ++ mktemp + local LAST_ERR=/tmp/tmp.5bzRfuwcXO + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'clusterissuer' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.qiUHlTJvOe + cat /tmp/tmp.5bzRfuwcXO error: the server doesn't have a resource type "clusterissuer" + sleep 0 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.qiUHlTJvOe + cat /tmp/tmp.5bzRfuwcXO error: the server doesn't have a resource type "clusterissuer" + sleep 4 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete clusterissuer --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.qiUHlTJvOe + cat /tmp/tmp.5bzRfuwcXO error: the server doesn't have a resource type "clusterissuer" + sleep 8 + cat /tmp/tmp.qiUHlTJvOe + cat /tmp/tmp.5bzRfuwcXO error: the server doesn't have a resource type "clusterissuer" + rm /tmp/tmp.qiUHlTJvOe /tmp/tmp.5bzRfuwcXO + return 1 + : + kubectl_bin -n cert-manager delete certificate --all ++ mktemp + local LAST_OUT=/tmp/tmp.JEQnmlu44O ++ mktemp + local LAST_ERR=/tmp/tmp.OITKaHG8RV + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'delete' 'certificate' '--all') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.JEQnmlu44O + cat /tmp/tmp.OITKaHG8RV error: the server doesn't have a resource type "certificate" + sleep 0 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.JEQnmlu44O + cat /tmp/tmp.OITKaHG8RV error: the server doesn't have a resource type "certificate" + sleep 4 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager delete certificate --all + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.JEQnmlu44O + cat /tmp/tmp.OITKaHG8RV error: the server doesn't have a resource type "certificate" + sleep 8 + cat /tmp/tmp.JEQnmlu44O + cat /tmp/tmp.OITKaHG8RV error: the server doesn't have a resource type "certificate" + rm /tmp/tmp.JEQnmlu44O /tmp/tmp.OITKaHG8RV + return 1 + : + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.Aoht8stejf ++ mktemp + local LAST_ERR=/tmp/tmp.23R1tqEGSK + local exit_status=0 + local timeout=4 + args=('create' 'namespace' 'cert-manager') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Aoht8stejf namespace/cert-manager created + cat /tmp/tmp.23R1tqEGSK + rm /tmp/tmp.Aoht8stejf /tmp/tmp.23R1tqEGSK + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.1Q4D4DH8z3 ++ mktemp + local LAST_ERR=/tmp/tmp.1plmkOevBZ + local exit_status=0 + local timeout=4 + args=('label' 'namespace' 'cert-manager' 'certmanager.k8s.io/disable-validation=true') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1Q4D4DH8z3 namespace/cert-manager labeled + cat /tmp/tmp.1plmkOevBZ + rm /tmp/tmp.1Q4D4DH8z3 /tmp/tmp.1plmkOevBZ + return 0 + curl -sL https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml + add_arch_to_podspecs - + arch_scheduling_enabled + kubectl_bin apply -f - --validate=false + [[ -n '' ]] + cat - + return namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created clusterrole.rbac.authorization.k8s.io/cert-manager-view created clusterrole.rbac.authorization.k8s.io/cert-manager-edit created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created role.rbac.authorization.k8s.io/cert-manager:leaderelection created role.rbac.authorization.k8s.io/cert-manager-tokenrequest created role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager-cainjector created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created + kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready ++ mktemp + local LAST_OUT=/tmp/tmp.B8jUww8PO6 ++ mktemp + local LAST_ERR=/tmp/tmp.rtqnDy9SyC + local exit_status=0 + local timeout=4 + args=('-n' 'cert-manager' 'wait' 'pod' '-l' 'app.kubernetes.io/instance=cert-manager' '--for=condition=ready') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.B8jUww8PO6 pod/cert-manager-559d798845-jxsw5 condition met pod/cert-manager-cainjector-64958d9c7c-k2dvz condition met pod/cert-manager-webhook-7fb6f99b56-fzt7s condition met + cat /tmp/tmp.rtqnDy9SyC + rm /tmp/tmp.B8jUww8PO6 /tmp/tmp.rtqnDy9SyC + return 0 + sleep 120 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.WB8tJVAuze ++ mktemp + local LAST_ERR=/tmp/tmp.e1VYGKmXOG + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WB8tJVAuze secret/some-users created + cat /tmp/tmp.e1VYGKmXOG + rm /tmp/tmp.WB8tJVAuze /tmp/tmp.e1VYGKmXOG + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0F93iYWgch ++ mktemp + local LAST_ERR=/tmp/tmp.fwn18aHp95 + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0F93iYWgch deployment.apps/psmdb-client created + cat /tmp/tmp.fwn18aHp95 + rm /tmp/tmp.0F93iYWgch /tmp/tmp.fwn18aHp95 + return 0 + deploy_cmctl + local service_account=cmctl + /usr/sbin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml + yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.oIwj0Kls4i ++ mktemp + local LAST_ERR=/tmp/tmp.MuAUcH0VIg + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oIwj0Kls4i role.rbac.authorization.k8s.io/cmctl created serviceaccount/cmctl created rolebinding.rbac.authorization.k8s.io/service-account-cmctl created + cat /tmp/tmp.MuAUcH0VIg + rm /tmp/tmp.oIwj0Kls4i /tmp/tmp.MuAUcH0VIg + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cmctl.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7Aiw71Kfb8 ++ mktemp + local LAST_ERR=/tmp/tmp.7rJxbM01Cr + local exit_status=0 + local timeout=4 + args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cmctl.yml') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cmctl.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.7Aiw71Kfb8 deployment.apps/cmctl created + cat /tmp/tmp.7rJxbM01Cr + rm /tmp/tmp.7Aiw71Kfb8 /tmp/tmp.7rJxbM01Cr + return 0 + desc 'create first PSMDB cluster 1.21.2 some-name' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster 1.21.2 some-name ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-78121c567"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + local LAST_OUT=/tmp/tmp.yqF8MWR59T + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"' + /usr/sbin/sed -e s/NAME_SPACE/upgrade-consistency-sharded-tls-2846/g + [[ -n docker.io/perconalab/fluentbit:main-logcollector ]] + IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector + yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)' + add_arch_to_psmdb - + arch_scheduling_enabled + [[ -n '' ]] + cat - + local LAST_ERR=/tmp/tmp.UpycY6yIlg + local exit_status=0 + local timeout=4 + args=('apply' '-f' '-') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + return + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yqF8MWR59T perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.UpycY6yIlg + rm /tmp/tmp.yqF8MWR59T /tmp/tmp.UpycY6yIlg + return 0 + desc 'check if Pod started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.............OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready...............OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EzGvuvRrFL +++ mktemp ++ local LAST_ERR=/tmp/tmp.l6pGHkPJCI ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EzGvuvRrFL ++ cat /tmp/tmp.l6pGHkPJCI ++ rm /tmp/tmp.EzGvuvRrFL /tmp/tmp.l6pGHkPJCI ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.............OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LXyz1kMFkM +++ mktemp ++ local LAST_ERR=/tmp/tmp.cICn9jyOYk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LXyz1kMFkM ++ cat /tmp/tmp.cICn9jyOYk ++ rm /tmp/tmp.LXyz1kMFkM /tmp/tmp.cICn9jyOYk ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JruKGzis4H +++ mktemp ++ local LAST_ERR=/tmp/tmp.jsbxFvZjak ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JruKGzis4H ++ cat /tmp/tmp.jsbxFvZjak ++ rm /tmp/tmp.JruKGzis4H /tmp/tmp.jsbxFvZjak ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready............................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1NsBkd9vcL +++ mktemp ++ local LAST_ERR=/tmp/tmp.XikQO5PSco ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1NsBkd9vcL ++ cat /tmp/tmp.XikQO5PSco ++ rm /tmp/tmp.1NsBkd9vcL /tmp/tmp.XikQO5PSco ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C8J4Sb0s6n +++ mktemp ++ local LAST_ERR=/tmp/tmp.SiATM5vZNh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C8J4Sb0s6n ++ cat /tmp/tmp.SiATM5vZNh ++ rm /tmp/tmp.C8J4Sb0s6n /tmp/tmp.SiATM5vZNh ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.70xWNwQHvH +++ mktemp ++ local LAST_ERR=/tmp/tmp.1WfbcS2DGz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.70xWNwQHvH ++ cat /tmp/tmp.1WfbcS2DGz ++ rm /tmp/tmp.70xWNwQHvH /tmp/tmp.1WfbcS2DGz ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iyFxVe3N2r +++ mktemp ++ local LAST_ERR=/tmp/tmp.HSCsZ2DGxD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iyFxVe3N2r ++ cat /tmp/tmp.HSCsZ2DGxD ++ rm /tmp/tmp.iyFxVe3N2r /tmp/tmp.HSCsZ2DGxD ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WYyjHtYFD5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RL4iOGsD3B ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WYyjHtYFD5 ++ cat /tmp/tmp.RL4iOGsD3B ++ rm /tmp/tmp.WYyjHtYFD5 /tmp/tmp.RL4iOGsD3B ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9HiQRrTIhn +++ mktemp ++ local LAST_ERR=/tmp/tmp.MrykaBMJ4D ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9HiQRrTIhn ++ cat /tmp/tmp.MrykaBMJ4D ++ rm /tmp/tmp.9HiQRrTIhn /tmp/tmp.MrykaBMJ4D ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mlCuMOFVUY +++ mktemp ++ local LAST_ERR=/tmp/tmp.NfqMWHhqxm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mlCuMOFVUY ++ cat /tmp/tmp.NfqMWHhqxm ++ rm /tmp/tmp.mlCuMOFVUY /tmp/tmp.NfqMWHhqxm ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 1 statefulset some-name-rs0 + local generation=1 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.400kSzwumJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.mKuPo86PVP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.400kSzwumJ ++ cat /tmp/tmp.mKuPo86PVP ++ rm /tmp/tmp.400kSzwumJ /tmp/tmp.mKuPo86PVP ++ return 0 + current_generation=1 + [[ 1 != 1 ]] + compare_generation 1 statefulset some-name-cfg + local generation=1 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vxJQddiEAe +++ mktemp ++ local LAST_ERR=/tmp/tmp.UABvLV0Tgf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vxJQddiEAe ++ cat /tmp/tmp.UABvLV0Tgf ++ rm /tmp/tmp.vxJQddiEAe /tmp/tmp.UABvLV0Tgf ++ return 0 + current_generation=1 + [[ 1 != 1 ]] + sleep 20 + desc 'check if Pod started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YFmAykTTNI +++ mktemp ++ local LAST_ERR=/tmp/tmp.VlmlfHreiU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YFmAykTTNI ++ cat /tmp/tmp.VlmlfHreiU ++ rm /tmp/tmp.YFmAykTTNI /tmp/tmp.VlmlfHreiU ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.O51xXiSiIN +++ mktemp ++ local LAST_ERR=/tmp/tmp.dJV80wZGIe ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.O51xXiSiIN ++ cat /tmp/tmp.dJV80wZGIe ++ rm /tmp/tmp.O51xXiSiIN /tmp/tmp.dJV80wZGIe ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xRrtBVHBVd +++ mktemp ++ local LAST_ERR=/tmp/tmp.mwLzvHmKWf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xRrtBVHBVd ++ cat /tmp/tmp.mwLzvHmKWf ++ rm /tmp/tmp.xRrtBVHBVd /tmp/tmp.mwLzvHmKWf ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JkTh3Ljp5M +++ mktemp ++ local LAST_ERR=/tmp/tmp.OJkzWnCNoJ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JkTh3Ljp5M ++ cat /tmp/tmp.OJkzWnCNoJ ++ rm /tmp/tmp.JkTh3Ljp5M /tmp/tmp.OJkzWnCNoJ ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7p0bQwYlqD +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tg0P3ub7Cj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7p0bQwYlqD ++ cat /tmp/tmp.Tg0P3ub7Cj ++ rm /tmp/tmp.7p0bQwYlqD /tmp/tmp.Tg0P3ub7Cj ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kfqR4IQwQR +++ mktemp ++ local LAST_ERR=/tmp/tmp.TWt4RlEQcS ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kfqR4IQwQR ++ cat /tmp/tmp.TWt4RlEQcS ++ rm /tmp/tmp.kfqR4IQwQR /tmp/tmp.TWt4RlEQcS ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EgGkzfo0MW +++ mktemp ++ local LAST_ERR=/tmp/tmp.1IrAi0p0jf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EgGkzfo0MW ++ cat /tmp/tmp.1IrAi0p0jf ++ rm /tmp/tmp.EgGkzfo0MW /tmp/tmp.1IrAi0p0jf ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YqfyV9bOyP +++ mktemp ++ local LAST_ERR=/tmp/tmp.GsZp4VqK77 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YqfyV9bOyP ++ cat /tmp/tmp.GsZp4VqK77 ++ rm /tmp/tmp.YqfyV9bOyP /tmp/tmp.GsZp4VqK77 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.07nISqC4Ws +++ mktemp ++ local LAST_ERR=/tmp/tmp.P5gv2blqmU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.07nISqC4Ws ++ cat /tmp/tmp.P5gv2blqmU ++ rm /tmp/tmp.07nISqC4Ws /tmp/tmp.P5gv2blqmU ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PI47uK41u4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ChodC8ma2z ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PI47uK41u4 ++ cat /tmp/tmp.ChodC8ma2z ++ rm /tmp/tmp.PI47uK41u4 /tmp/tmp.ChodC8ma2z ++ return 0 + [[ ready == ready ]] + echo .OK .OK + renew_certificate some-name-ssl + certificate=some-name-ssl + wait_certificate some-name-ssl + certificate=some-name-ssl + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:22.516986 2446771 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254222143719999&timeoutSeconds=465&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:24.773244 2447000 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254221885289000&timeoutSeconds=313&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:27.082517 2447225 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254226885317000&timeoutSeconds=398&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:29.499938 2447512 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254226885317000&timeoutSeconds=344&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:31.745226 2447678 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254226885317000&timeoutSeconds=567&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:34.085140 2447861 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254232195473000&timeoutSeconds=352&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:36.386381 2448183 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254232195473000&timeoutSeconds=540&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:38.633071 2448469 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254237195522000&timeoutSeconds=492&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:40.929559 2448793 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254237195522000&timeoutSeconds=481&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:10:43.273897 2449171 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784254241885389000&timeoutSeconds=595&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + desc 'renew some-name-ssl' + set +o xtrace ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- + local pod_name ++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZAud7pmM42 +++ mktemp ++ local LAST_ERR=/tmp/tmp.jZ2KnyHp1Y ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=cmctl' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZAud7pmM42 ++ cat /tmp/tmp.jZ2KnyHp1Y ++ rm /tmp/tmp.ZAud7pmM42 /tmp/tmp.jZ2KnyHp1Y ++ return 0 + pod_name=cmctl-5bdf95f5b4-xm7mp + local revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iyuQykrJw7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.TgxvtZXndD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.iyuQykrJw7 ++ cat /tmp/tmp.TgxvtZXndD ++ rm /tmp/tmp.iyuQykrJw7 /tmp/tmp.TgxvtZXndD ++ return 0 + revision=1 + kubectl_bin exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl ++ mktemp + local LAST_OUT=/tmp/tmp.3nhTLVH8Ht ++ mktemp + local LAST_ERR=/tmp/tmp.iGv8266Z92 + local exit_status=0 + local timeout=4 + args=('exec' 'cmctl-5bdf95f5b4-xm7mp' '--' '/tmp/cmctl' 'renew' 'some-name-ssl') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3nhTLVH8Ht Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-2846/some-name-ssl + cat /tmp/tmp.iGv8266Z92 + rm /tmp/tmp.3nhTLVH8Ht /tmp/tmp.iGv8266Z92 + return 0 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.86ji3NtEpR +++ mktemp ++ local LAST_ERR=/tmp/tmp.bEx9dhQ0AN ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.86ji3NtEpR ++ cat /tmp/tmp.bEx9dhQ0AN ++ rm /tmp/tmp.86ji3NtEpR /tmp/tmp.bEx9dhQ0AN ++ return 0 + new_revision=2 + '[' 2 == 2 ']' + break + sleep 20 + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mvwteQ2WZW +++ mktemp ++ local LAST_ERR=/tmp/tmp.d0Q8F8FYXp ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mvwteQ2WZW ++ cat /tmp/tmp.d0Q8F8FYXp ++ rm /tmp/tmp.mvwteQ2WZW /tmp/tmp.d0Q8F8FYXp ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7qZ8BjaCHd +++ mktemp ++ local LAST_ERR=/tmp/tmp.30mMWbziaf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7qZ8BjaCHd ++ cat /tmp/tmp.30mMWbziaf ++ rm /tmp/tmp.7qZ8BjaCHd /tmp/tmp.30mMWbziaf ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0Z77ZTQOyI +++ mktemp ++ local LAST_ERR=/tmp/tmp.SVpdrqeWyB ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0Z77ZTQOyI ++ cat /tmp/tmp.SVpdrqeWyB ++ rm /tmp/tmp.0Z77ZTQOyI /tmp/tmp.SVpdrqeWyB ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready............................................................................................................................................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OKlISdigP2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zgxssx4XW1 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OKlISdigP2 ++ cat /tmp/tmp.Zgxssx4XW1 ++ rm /tmp/tmp.OKlISdigP2 /tmp/tmp.Zgxssx4XW1 ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Kjzhi2kSQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.wSax5LPVm3 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.6Kjzhi2kSQ ++ cat /tmp/tmp.wSax5LPVm3 ++ rm /tmp/tmp.6Kjzhi2kSQ /tmp/tmp.wSax5LPVm3 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bh3sG2zcAN +++ mktemp ++ local LAST_ERR=/tmp/tmp.ikO2HIm38B ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bh3sG2zcAN ++ cat /tmp/tmp.ikO2HIm38B ++ rm /tmp/tmp.bh3sG2zcAN /tmp/tmp.ikO2HIm38B ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jVir324w6X +++ mktemp ++ local LAST_ERR=/tmp/tmp.hG4WWp18Dz ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jVir324w6X ++ cat /tmp/tmp.hG4WWp18Dz ++ rm /tmp/tmp.jVir324w6X /tmp/tmp.hG4WWp18Dz ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F4XMGICG1G +++ mktemp ++ local LAST_ERR=/tmp/tmp.3tFBkMeOcO ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.F4XMGICG1G ++ cat /tmp/tmp.3tFBkMeOcO ++ rm /tmp/tmp.F4XMGICG1G /tmp/tmp.3tFBkMeOcO ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TXtnL9Zkvd +++ mktemp ++ local LAST_ERR=/tmp/tmp.9hSJQyceSM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TXtnL9Zkvd ++ cat /tmp/tmp.9hSJQyceSM ++ rm /tmp/tmp.TXtnL9Zkvd /tmp/tmp.9hSJQyceSM ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tgt76HA74U +++ mktemp ++ local LAST_ERR=/tmp/tmp.Eb7Y3hbJyQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Tgt76HA74U ++ cat /tmp/tmp.Eb7Y3hbJyQ ++ rm /tmp/tmp.Tgt76HA74U /tmp/tmp.Eb7Y3hbJyQ ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 2 statefulset some-name-rs0 + local generation=2 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HE7shppzLv +++ mktemp ++ local LAST_ERR=/tmp/tmp.1LVIqAjW0Z ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HE7shppzLv ++ cat /tmp/tmp.1LVIqAjW0Z ++ rm /tmp/tmp.HE7shppzLv /tmp/tmp.1LVIqAjW0Z ++ return 0 + current_generation=2 + [[ 2 != 2 ]] + compare_generation 2 statefulset some-name-cfg + local generation=2 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s5q6SMTrwR +++ mktemp ++ local LAST_ERR=/tmp/tmp.XaBRYNngVj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s5q6SMTrwR ++ cat /tmp/tmp.XaBRYNngVj ++ rm /tmp/tmp.s5q6SMTrwR /tmp/tmp.XaBRYNngVj ++ return 0 + current_generation=2 + [[ 2 != 2 ]] + renew_certificate some-name-ssl-internal + certificate=some-name-ssl-internal + wait_certificate some-name-ssl-internal + certificate=some-name-ssl-internal + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:10.365215 2493015 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254628140564000&timeoutSeconds=491&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:12.709169 2493210 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254628139973000&timeoutSeconds=586&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:15.040502 2493391 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254633139993000&timeoutSeconds=583&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:17.347659 2493613 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254633140570000&timeoutSeconds=577&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:19.646377 2493907 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254638140581000&timeoutSeconds=328&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:21.905305 2494126 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254638140581000&timeoutSeconds=597&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:24.158268 2494417 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254643140082000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:26.608019 2494715 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254643140594000&timeoutSeconds=464&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:28.893967 2494976 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254648140602000&timeoutSeconds=466&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:17:31.236994 2495202 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784254648140272000&timeoutSeconds=469&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + desc 'renew some-name-ssl-internal' + set +o xtrace ----------------------------------------------------------------------------------- renew some-name-ssl-internal ----------------------------------------------------------------------------------- + local pod_name ++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nxLY0y0Maj +++ mktemp ++ local LAST_ERR=/tmp/tmp.7aNMx0sOYp ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=cmctl' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nxLY0y0Maj ++ cat /tmp/tmp.7aNMx0sOYp ++ rm /tmp/tmp.nxLY0y0Maj /tmp/tmp.7aNMx0sOYp ++ return 0 + pod_name=cmctl-5bdf95f5b4-xm7mp + local revision ++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mxH6v0AhP1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DfmS2Qcrhe ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl-internal' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mxH6v0AhP1 ++ cat /tmp/tmp.DfmS2Qcrhe ++ rm /tmp/tmp.mxH6v0AhP1 /tmp/tmp.DfmS2Qcrhe ++ return 0 + revision=1 + kubectl_bin exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl-internal ++ mktemp + local LAST_OUT=/tmp/tmp.SEtn1tM4Ac ++ mktemp + local LAST_ERR=/tmp/tmp.2M1tEOUjnm + local exit_status=0 + local timeout=4 + args=('exec' 'cmctl-5bdf95f5b4-xm7mp' '--' '/tmp/cmctl' 'renew' 'some-name-ssl-internal') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl-internal + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SEtn1tM4Ac Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-2846/some-name-ssl-internal + cat /tmp/tmp.2M1tEOUjnm + rm /tmp/tmp.SEtn1tM4Ac /tmp/tmp.2M1tEOUjnm + return 0 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1xlHEjDjWQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.sSidRHKLXw ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl-internal' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1xlHEjDjWQ ++ cat /tmp/tmp.sSidRHKLXw ++ rm /tmp/tmp.1xlHEjDjWQ /tmp/tmp.sSidRHKLXw ++ return 0 + new_revision=2 + '[' 2 == 2 ']' + break + sleep 20 + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CVB1qON0Od +++ mktemp ++ local LAST_ERR=/tmp/tmp.pMYUHvYdRK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CVB1qON0Od ++ cat /tmp/tmp.pMYUHvYdRK ++ rm /tmp/tmp.CVB1qON0Od /tmp/tmp.pMYUHvYdRK ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qG9I6ssDjU +++ mktemp ++ local LAST_ERR=/tmp/tmp.eofhnKRqO7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qG9I6ssDjU ++ cat /tmp/tmp.eofhnKRqO7 ++ rm /tmp/tmp.qG9I6ssDjU /tmp/tmp.eofhnKRqO7 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VlROTmpuSU +++ mktemp ++ local LAST_ERR=/tmp/tmp.2Iq87MIQ8r ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VlROTmpuSU ++ cat /tmp/tmp.2Iq87MIQ8r ++ rm /tmp/tmp.VlROTmpuSU /tmp/tmp.2Iq87MIQ8r ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready................................................................................................................................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Lf7hoqBMg1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mawUSn5zEj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Lf7hoqBMg1 ++ cat /tmp/tmp.mawUSn5zEj ++ rm /tmp/tmp.Lf7hoqBMg1 /tmp/tmp.mawUSn5zEj ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zo9xWVARyM +++ mktemp ++ local LAST_ERR=/tmp/tmp.CSFFtOjI7w ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zo9xWVARyM ++ cat /tmp/tmp.CSFFtOjI7w ++ rm /tmp/tmp.zo9xWVARyM /tmp/tmp.CSFFtOjI7w ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BrO4DCwOGs +++ mktemp ++ local LAST_ERR=/tmp/tmp.h0QGlSqGSQ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BrO4DCwOGs ++ cat /tmp/tmp.h0QGlSqGSQ ++ rm /tmp/tmp.BrO4DCwOGs /tmp/tmp.h0QGlSqGSQ ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8r5tqjfIMc +++ mktemp ++ local LAST_ERR=/tmp/tmp.I4dsCQGYbG ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8r5tqjfIMc ++ cat /tmp/tmp.I4dsCQGYbG ++ rm /tmp/tmp.8r5tqjfIMc /tmp/tmp.I4dsCQGYbG ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.08kS98q2gM +++ mktemp ++ local LAST_ERR=/tmp/tmp.a2XjWhybL7 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.08kS98q2gM ++ cat /tmp/tmp.a2XjWhybL7 ++ rm /tmp/tmp.08kS98q2gM /tmp/tmp.a2XjWhybL7 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bGkEFRYems +++ mktemp ++ local LAST_ERR=/tmp/tmp.9QnVt6lWqM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bGkEFRYems ++ cat /tmp/tmp.9QnVt6lWqM ++ rm /tmp/tmp.bGkEFRYems /tmp/tmp.9QnVt6lWqM ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LzkDzm2gte +++ mktemp ++ local LAST_ERR=/tmp/tmp.mO0XgdxO8n ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LzkDzm2gte ++ cat /tmp/tmp.mO0XgdxO8n ++ rm /tmp/tmp.LzkDzm2gte /tmp/tmp.mO0XgdxO8n ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 3 statefulset some-name-rs0 + local generation=3 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EefPbyPMRu +++ mktemp ++ local LAST_ERR=/tmp/tmp.jszYm9n6oL ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EefPbyPMRu ++ cat /tmp/tmp.jszYm9n6oL ++ rm /tmp/tmp.EefPbyPMRu /tmp/tmp.jszYm9n6oL ++ return 0 + current_generation=3 + [[ 3 != 3 ]] + compare_generation 3 statefulset some-name-cfg + local generation=3 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JfQnuqZqhV +++ mktemp ++ local LAST_ERR=/tmp/tmp.ty44Y2pOhH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JfQnuqZqhV ++ cat /tmp/tmp.ty44Y2pOhH ++ rm /tmp/tmp.JfQnuqZqhV /tmp/tmp.ty44Y2pOhH ++ return 0 + current_generation=3 + [[ 3 != 3 ]] + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl service/some-name-rs0 -1212 + local resource=service/some-name-rs0 + local postfix=-1212 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml + local new_result=/tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.tJF8UZ02eN ++ mktemp + local LAST_ERR=/tmp/tmp.fOKdAuIZnc + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tJF8UZ02eN + cat /tmp/tmp.fOKdAuIZnc + rm /tmp/tmp.tJF8UZ02eN /tmp/tmp.fOKdAuIZnc + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + log 'compare_kubectl: service/some-name-rs0 OK' + set +o xtrace [2026-07-17T02:23:29+0000] compare_kubectl: service/some-name-rs0 OK + compare_kubectl service/some-name-cfg -1212 + local resource=service/some-name-cfg + local postfix=-1212 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml + local new_result=/tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/some-name-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.M2Fob04Sk7 ++ mktemp + local LAST_ERR=/tmp/tmp.KiaKljmCHC + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/some-name-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.M2Fob04Sk7 + cat /tmp/tmp.KiaKljmCHC + rm /tmp/tmp.M2Fob04Sk7 /tmp/tmp.KiaKljmCHC + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + log 'compare_kubectl: service/some-name-cfg OK' + set +o xtrace [2026-07-17T02:23:30+0000] compare_kubectl: service/some-name-cfg OK + compare_kubectl statefulset/some-name-rs0 -1212 + local resource=statefulset/some-name-rs0 + local postfix=-1212 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml + local new_result=/tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.ahrCV5Kkuv ++ mktemp + local LAST_ERR=/tmp/tmp.veaSGGjTAc + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ahrCV5Kkuv + cat /tmp/tmp.veaSGGjTAc + rm /tmp/tmp.ahrCV5Kkuv /tmp/tmp.veaSGGjTAc + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-07-17T02:23:31+0000] compare_kubectl: statefulset/some-name-rs0 OK + compare_kubectl statefulset/some-name-cfg -1212 + local resource=statefulset/some-name-cfg + local postfix=-1212 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml + local new_result=/tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.n2ZCguVVEK ++ mktemp + local LAST_ERR=/tmp/tmp.LG8fFxmOR0 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.n2ZCguVVEK + cat /tmp/tmp.LG8fFxmOR0 + rm /tmp/tmp.n2ZCguVVEK /tmp/tmp.LG8fFxmOR0 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + log 'compare_kubectl: statefulset/some-name-cfg OK' + set +o xtrace [2026-07-17T02:23:32+0000] compare_kubectl: statefulset/some-name-cfg OK + desc 'test 1.22.0' + set +o xtrace ----------------------------------------------------------------------------------- test 1.22.0 ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=merge --patch $'{\n "spec": {"crVersion":"1.22.0"}\n }' ++ mktemp + local LAST_OUT=/tmp/tmp.UVcs8DzNMG ++ mktemp + local LAST_ERR=/tmp/tmp.26wiwo3qIP + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=merge' '--patch' '{ "spec": {"crVersion":"1.22.0"} }') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=merge --patch $'{\n "spec": {"crVersion":"1.22.0"}\n }' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UVcs8DzNMG perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.26wiwo3qIP + rm /tmp/tmp.UVcs8DzNMG /tmp/tmp.26wiwo3qIP + return 0 + sleep 20 + desc 'check if Pod started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LvhbnSjbmX +++ mktemp ++ local LAST_ERR=/tmp/tmp.CdYSFoJgXM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LvhbnSjbmX ++ cat /tmp/tmp.CdYSFoJgXM ++ rm /tmp/tmp.LvhbnSjbmX /tmp/tmp.CdYSFoJgXM ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QEDTnUqhfK +++ mktemp ++ local LAST_ERR=/tmp/tmp.iIc0wwOZmc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QEDTnUqhfK ++ cat /tmp/tmp.iIc0wwOZmc ++ rm /tmp/tmp.QEDTnUqhfK /tmp/tmp.iIc0wwOZmc ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.COBiLhRKM8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.e83SBG5JQm ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.COBiLhRKM8 ++ cat /tmp/tmp.e83SBG5JQm ++ rm /tmp/tmp.COBiLhRKM8 /tmp/tmp.e83SBG5JQm ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready.............................................................................................. + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sopkf7HHm8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.et0eCFSDym ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sopkf7HHm8 ++ cat /tmp/tmp.et0eCFSDym ++ rm /tmp/tmp.sopkf7HHm8 /tmp/tmp.et0eCFSDym ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ilSXZFcfa8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.E7GCbxbVqP ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ilSXZFcfa8 ++ cat /tmp/tmp.E7GCbxbVqP ++ rm /tmp/tmp.ilSXZFcfa8 /tmp/tmp.E7GCbxbVqP ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Lh6E6zmY4V +++ mktemp ++ local LAST_ERR=/tmp/tmp.sAocb6qX7u ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Lh6E6zmY4V ++ cat /tmp/tmp.sAocb6qX7u ++ rm /tmp/tmp.Lh6E6zmY4V /tmp/tmp.sAocb6qX7u ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BRGXqd9Ysg +++ mktemp ++ local LAST_ERR=/tmp/tmp.TRsMSPy6oC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BRGXqd9Ysg ++ cat /tmp/tmp.TRsMSPy6oC ++ rm /tmp/tmp.BRGXqd9Ysg /tmp/tmp.TRsMSPy6oC ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bt04y8dTl0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.8KCP2XPfGM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bt04y8dTl0 ++ cat /tmp/tmp.8KCP2XPfGM ++ rm /tmp/tmp.bt04y8dTl0 /tmp/tmp.8KCP2XPfGM ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TvWbZdaIsC +++ mktemp ++ local LAST_ERR=/tmp/tmp.RjR69XwHUW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TvWbZdaIsC ++ cat /tmp/tmp.RjR69XwHUW ++ rm /tmp/tmp.TvWbZdaIsC /tmp/tmp.RjR69XwHUW ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2yhF9uuJBs +++ mktemp ++ local LAST_ERR=/tmp/tmp.sI7cEZYZzA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2yhF9uuJBs ++ cat /tmp/tmp.sI7cEZYZzA ++ rm /tmp/tmp.2yhF9uuJBs /tmp/tmp.sI7cEZYZzA ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 4 statefulset some-name-rs0 + local generation=4 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.a9tY7iLfhv +++ mktemp ++ local LAST_ERR=/tmp/tmp.LQHwHajCYi ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.a9tY7iLfhv ++ cat /tmp/tmp.LQHwHajCYi ++ rm /tmp/tmp.a9tY7iLfhv /tmp/tmp.LQHwHajCYi ++ return 0 + current_generation=4 + [[ 4 != 4 ]] + compare_generation 4 statefulset some-name-cfg + local generation=4 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.baxiRZYiFT +++ mktemp ++ local LAST_ERR=/tmp/tmp.oY1WS5MiCj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.baxiRZYiFT ++ cat /tmp/tmp.oY1WS5MiCj ++ rm /tmp/tmp.baxiRZYiFT /tmp/tmp.oY1WS5MiCj ++ return 0 + current_generation=4 + [[ 4 != 4 ]] + renew_certificate some-name-ssl + certificate=some-name-ssl + wait_certificate some-name-ssl + certificate=some-name-ssl + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:11.247793 2550360 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255286394109000&timeoutSeconds=464&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:13.543421 2550630 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255291394158000&timeoutSeconds=579&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:15.926362 2550857 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255291393749000&timeoutSeconds=426&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:18.287416 2551059 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255296394163000&timeoutSeconds=588&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:20.605276 2551246 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255296393765000&timeoutSeconds=525&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:22.872911 2551385 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255301393861000&timeoutSeconds=569&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:25.173873 2551546 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255301393861000&timeoutSeconds=329&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:27.496039 2551693 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255306394327000&timeoutSeconds=468&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:29.818177 2551830 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255306393868000&timeoutSeconds=509&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:28:32.087164 2551993 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784255311394073000&timeoutSeconds=490&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + desc 'renew some-name-ssl' + set +o xtrace ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- + local pod_name ++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mYLfdidKCp +++ mktemp ++ local LAST_ERR=/tmp/tmp.BmmRWCfjhA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=cmctl' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mYLfdidKCp ++ cat /tmp/tmp.BmmRWCfjhA ++ rm /tmp/tmp.mYLfdidKCp /tmp/tmp.BmmRWCfjhA ++ return 0 + pod_name=cmctl-5bdf95f5b4-xm7mp + local revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EGLqzwSCQJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.eP7c2MzW2Z ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EGLqzwSCQJ ++ cat /tmp/tmp.eP7c2MzW2Z ++ rm /tmp/tmp.EGLqzwSCQJ /tmp/tmp.eP7c2MzW2Z ++ return 0 + revision=2 + kubectl_bin exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl ++ mktemp + local LAST_OUT=/tmp/tmp.FuJ4BkleGv ++ mktemp + local LAST_ERR=/tmp/tmp.XpPEemok3R + local exit_status=0 + local timeout=4 + args=('exec' 'cmctl-5bdf95f5b4-xm7mp' '--' '/tmp/cmctl' 'renew' 'some-name-ssl') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FuJ4BkleGv Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-2846/some-name-ssl + cat /tmp/tmp.XpPEemok3R + rm /tmp/tmp.FuJ4BkleGv /tmp/tmp.XpPEemok3R + return 0 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uudYJJJsbF +++ mktemp ++ local LAST_ERR=/tmp/tmp.oXxAke8lVM ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uudYJJJsbF ++ cat /tmp/tmp.oXxAke8lVM ++ rm /tmp/tmp.uudYJJJsbF /tmp/tmp.oXxAke8lVM ++ return 0 + new_revision=3 + '[' 3 == 3 ']' + break + sleep 20 + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8BNU2FfWIz +++ mktemp ++ local LAST_ERR=/tmp/tmp.1R4IQHtJgY ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8BNU2FfWIz ++ cat /tmp/tmp.1R4IQHtJgY ++ rm /tmp/tmp.8BNU2FfWIz /tmp/tmp.1R4IQHtJgY ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xRCe9IG1Pv +++ mktemp ++ local LAST_ERR=/tmp/tmp.K4HCHah3l8 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xRCe9IG1Pv ++ cat /tmp/tmp.K4HCHah3l8 ++ rm /tmp/tmp.xRCe9IG1Pv /tmp/tmp.K4HCHah3l8 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UdtIttxD2c +++ mktemp ++ local LAST_ERR=/tmp/tmp.wFkNTYM5aK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UdtIttxD2c ++ cat /tmp/tmp.wFkNTYM5aK ++ rm /tmp/tmp.UdtIttxD2c /tmp/tmp.wFkNTYM5aK ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready...................................................................................................................................................... + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vb2Eg8H8Ro +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ajv7kEtjjF ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vb2Eg8H8Ro ++ cat /tmp/tmp.Ajv7kEtjjF ++ rm /tmp/tmp.vb2Eg8H8Ro /tmp/tmp.Ajv7kEtjjF ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DvWjovaAol +++ mktemp ++ local LAST_ERR=/tmp/tmp.14IGQg9vvL ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DvWjovaAol ++ cat /tmp/tmp.14IGQg9vvL ++ rm /tmp/tmp.DvWjovaAol /tmp/tmp.14IGQg9vvL ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8HP3rpMdTN +++ mktemp ++ local LAST_ERR=/tmp/tmp.gMzFCqH8WU ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8HP3rpMdTN ++ cat /tmp/tmp.gMzFCqH8WU ++ rm /tmp/tmp.8HP3rpMdTN /tmp/tmp.gMzFCqH8WU ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s8tQ2kAgXD +++ mktemp ++ local LAST_ERR=/tmp/tmp.gJ25U1fbJu ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.s8tQ2kAgXD ++ cat /tmp/tmp.gJ25U1fbJu ++ rm /tmp/tmp.s8tQ2kAgXD /tmp/tmp.gJ25U1fbJu ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.odRL2w0MZy +++ mktemp ++ local LAST_ERR=/tmp/tmp.QN8WRjqnxf ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.odRL2w0MZy ++ cat /tmp/tmp.QN8WRjqnxf ++ rm /tmp/tmp.odRL2w0MZy /tmp/tmp.QN8WRjqnxf ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Eqx6pkUMud +++ mktemp ++ local LAST_ERR=/tmp/tmp.KzXvvsubhC ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Eqx6pkUMud ++ cat /tmp/tmp.KzXvvsubhC ++ rm /tmp/tmp.Eqx6pkUMud /tmp/tmp.KzXvvsubhC ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xhExSAIKii +++ mktemp ++ local LAST_ERR=/tmp/tmp.77TXGO6RHd ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xhExSAIKii ++ cat /tmp/tmp.77TXGO6RHd ++ rm /tmp/tmp.xhExSAIKii /tmp/tmp.77TXGO6RHd ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 5 statefulset some-name-rs0 + local generation=5 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sIkHVeHp9A +++ mktemp ++ local LAST_ERR=/tmp/tmp.olsRXGJV1q ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.sIkHVeHp9A ++ cat /tmp/tmp.olsRXGJV1q ++ rm /tmp/tmp.sIkHVeHp9A /tmp/tmp.olsRXGJV1q ++ return 0 + current_generation=5 + [[ 5 != 5 ]] + compare_generation 5 statefulset some-name-cfg + local generation=5 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wJJcIuwull +++ mktemp ++ local LAST_ERR=/tmp/tmp.jGjin2ZoQW ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.wJJcIuwull ++ cat /tmp/tmp.jGjin2ZoQW ++ rm /tmp/tmp.wJJcIuwull /tmp/tmp.jGjin2ZoQW ++ return 0 + current_generation=5 + [[ 5 != 5 ]] + renew_certificate some-name-ssl-internal + certificate=some-name-ssl-internal + wait_certificate some-name-ssl-internal + certificate=some-name-ssl-internal + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:45.026898 2565243 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255681759469000&timeoutSeconds=491&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:46.795044 2565266 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255681761764000&timeoutSeconds=406&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:48.548330 2565322 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255686761804000&timeoutSeconds=325&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:50.861562 2565383 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255686759592000&timeoutSeconds=387&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:52.615786 2565416 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255691759667000&timeoutSeconds=576&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:54.380077 2565442 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255691761844000&timeoutSeconds=386&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:56.149173 2565464 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255691759667000&timeoutSeconds=331&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:57.906008 2565488 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255696761892000&timeoutSeconds=573&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:34:59.808018 2565510 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255696761892000&timeoutSeconds=540&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s E0717 02:35:02.097456 2565535 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1784255701761944000&timeoutSeconds=511&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl-internal condition met + sleep 1 + desc 'renew some-name-ssl-internal' + set +o xtrace ----------------------------------------------------------------------------------- renew some-name-ssl-internal ----------------------------------------------------------------------------------- + local pod_name ++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VPa1SOyJZV +++ mktemp ++ local LAST_ERR=/tmp/tmp.LcpywvZhJ0 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=cmctl' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VPa1SOyJZV ++ cat /tmp/tmp.LcpywvZhJ0 ++ rm /tmp/tmp.VPa1SOyJZV /tmp/tmp.LcpywvZhJ0 ++ return 0 + pod_name=cmctl-5bdf95f5b4-xm7mp + local revision ++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hHfo3V98C5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.p9BqKKuWXO ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl-internal' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hHfo3V98C5 ++ cat /tmp/tmp.p9BqKKuWXO ++ rm /tmp/tmp.hHfo3V98C5 /tmp/tmp.p9BqKKuWXO ++ return 0 + revision=2 + kubectl_bin exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl-internal ++ mktemp + local LAST_OUT=/tmp/tmp.xaP209fWka ++ mktemp + local LAST_ERR=/tmp/tmp.Q5mTsAzLOt + local exit_status=0 + local timeout=4 + args=('exec' 'cmctl-5bdf95f5b4-xm7mp' '--' '/tmp/cmctl' 'renew' 'some-name-ssl-internal') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl-internal + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xaP209fWka Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-2846/some-name-ssl-internal + cat /tmp/tmp.Q5mTsAzLOt + rm /tmp/tmp.xaP209fWka /tmp/tmp.Q5mTsAzLOt + return 0 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Rvu8xdwyX0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yng16nqyJs ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl-internal' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Rvu8xdwyX0 ++ cat /tmp/tmp.yng16nqyJs ++ rm /tmp/tmp.Rvu8xdwyX0 /tmp/tmp.yng16nqyJs ++ return 0 + new_revision=3 + '[' 3 == 3 ']' + break + sleep 20 + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.na6XchHON0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dTHplzUUtK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.na6XchHON0 ++ cat /tmp/tmp.dTHplzUUtK ++ rm /tmp/tmp.na6XchHON0 /tmp/tmp.dTHplzUUtK ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BJea3VIznX +++ mktemp ++ local LAST_ERR=/tmp/tmp.u80c9e1l4g ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BJea3VIznX ++ cat /tmp/tmp.u80c9e1l4g ++ rm /tmp/tmp.BJea3VIznX /tmp/tmp.u80c9e1l4g ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tpwqcFGtxq +++ mktemp ++ local LAST_ERR=/tmp/tmp.9ayFtRcJpy ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tpwqcFGtxq ++ cat /tmp/tmp.9ayFtRcJpy ++ rm /tmp/tmp.tpwqcFGtxq /tmp/tmp.9ayFtRcJpy ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready................................................................................................................................................................ + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5V0ZqoEvKS +++ mktemp ++ local LAST_ERR=/tmp/tmp.E0okYsJGHn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5V0ZqoEvKS ++ cat /tmp/tmp.E0okYsJGHn ++ rm /tmp/tmp.5V0ZqoEvKS /tmp/tmp.E0okYsJGHn ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3PBrPQHsdH +++ mktemp ++ local LAST_ERR=/tmp/tmp.QwuuAyH1vs ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3PBrPQHsdH ++ cat /tmp/tmp.QwuuAyH1vs ++ rm /tmp/tmp.3PBrPQHsdH /tmp/tmp.QwuuAyH1vs ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BiYmqgcXS3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.H4Fh9BNvSh ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BiYmqgcXS3 ++ cat /tmp/tmp.H4Fh9BNvSh ++ rm /tmp/tmp.BiYmqgcXS3 /tmp/tmp.H4Fh9BNvSh ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aeEADT3ec8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mwem0mQJdK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aeEADT3ec8 ++ cat /tmp/tmp.Mwem0mQJdK ++ rm /tmp/tmp.aeEADT3ec8 /tmp/tmp.Mwem0mQJdK ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tVc5Nh81OP +++ mktemp ++ local LAST_ERR=/tmp/tmp.tLxtxdYssA ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tVc5Nh81OP ++ cat /tmp/tmp.tLxtxdYssA ++ rm /tmp/tmp.tVc5Nh81OP /tmp/tmp.tLxtxdYssA ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pAR5ZJVKqE +++ mktemp ++ local LAST_ERR=/tmp/tmp.I0JPk3rNvV ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pAR5ZJVKqE ++ cat /tmp/tmp.I0JPk3rNvV ++ rm /tmp/tmp.pAR5ZJVKqE /tmp/tmp.I0JPk3rNvV ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NospZA7Dk7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.hDN85Gy2l4 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NospZA7Dk7 ++ cat /tmp/tmp.hDN85Gy2l4 ++ rm /tmp/tmp.NospZA7Dk7 /tmp/tmp.hDN85Gy2l4 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 6 statefulset some-name-rs0 + local generation=6 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n9g44KvWHo +++ mktemp ++ local LAST_ERR=/tmp/tmp.4bx058oa6w ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.n9g44KvWHo ++ cat /tmp/tmp.4bx058oa6w ++ rm /tmp/tmp.n9g44KvWHo /tmp/tmp.4bx058oa6w ++ return 0 + current_generation=6 + [[ 6 != 6 ]] + compare_generation 6 statefulset some-name-cfg + local generation=6 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BAhao99EDS +++ mktemp ++ local LAST_ERR=/tmp/tmp.U3atIvovZJ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BAhao99EDS ++ cat /tmp/tmp.U3atIvovZJ ++ rm /tmp/tmp.BAhao99EDS /tmp/tmp.U3atIvovZJ ++ return 0 + current_generation=6 + [[ 6 != 6 ]] + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl service/some-name-rs0 -1220 + local resource=service/some-name-rs0 + local postfix=-1220 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml + local new_result=/tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.CecJedZsQL ++ mktemp + local LAST_ERR=/tmp/tmp.Bg0E7Wp5EH + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.CecJedZsQL + cat /tmp/tmp.Bg0E7Wp5EH + rm /tmp/tmp.CecJedZsQL /tmp/tmp.Bg0E7Wp5EH + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml /tmp/tmp.4wRom9JXCk/service_some-name-rs0.yml + log 'compare_kubectl: service/some-name-rs0 OK' + set +o xtrace [2026-07-17T02:40:31+0000] compare_kubectl: service/some-name-rs0 OK + compare_kubectl service/some-name-cfg -1220 + local resource=service/some-name-cfg + local postfix=-1220 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml + local new_result=/tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml service/some-name-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.m9T4IoSlP2 ++ mktemp + local LAST_ERR=/tmp/tmp.xS9ksISymv + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'service/some-name-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml service/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.m9T4IoSlP2 + cat /tmp/tmp.xS9ksISymv + rm /tmp/tmp.m9T4IoSlP2 /tmp/tmp.xS9ksISymv + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml /tmp/tmp.4wRom9JXCk/service_some-name-cfg.yml + log 'compare_kubectl: service/some-name-cfg OK' + set +o xtrace [2026-07-17T02:40:31+0000] compare_kubectl: service/some-name-cfg OK + compare_kubectl statefulset/some-name-rs0 -1220 + local resource=statefulset/some-name-rs0 + local postfix=-1220 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml + local new_result=/tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-rs0 + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.eEJtJetmQJ ++ mktemp + local LAST_ERR=/tmp/tmp.VuDNlz7YR9 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-rs0') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-rs0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.eEJtJetmQJ + cat /tmp/tmp.VuDNlz7YR9 + rm /tmp/tmp.eEJtJetmQJ /tmp/tmp.VuDNlz7YR9 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml /tmp/tmp.4wRom9JXCk/statefulset_some-name-rs0.yml + log 'compare_kubectl: statefulset/some-name-rs0 OK' + set +o xtrace [2026-07-17T02:40:32+0000] compare_kubectl: statefulset/some-name-rs0 OK + compare_kubectl statefulset/some-name-cfg -1220 + local resource=statefulset/some-name-cfg + local postfix=-1220 + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml + local new_result=/tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + [[ -n '' ]] + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."field.cattle.io/publicEndpoints") |\n\t\t\tdel(.metadata.annotations."metallb.io/ip-allocated-from-pool") |\n\t\t\tdel(.metadata.annotations."networking.gke.io/target-pool") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-2846", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.04MSJvOo1E ++ mktemp + local LAST_ERR=/tmp/tmp.3aUvIT76w8 + local exit_status=0 + local timeout=4 + args=('get' '-o' 'yaml' 'statefulset/some-name-cfg') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.04MSJvOo1E + cat /tmp/tmp.3aUvIT76w8 + rm /tmp/tmp.04MSJvOo1E /tmp/tmp.3aUvIT76w8 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + arch_scheduling_enabled + [[ -n '' ]] + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml /tmp/tmp.4wRom9JXCk/statefulset_some-name-cfg.yml + log 'compare_kubectl: statefulset/some-name-cfg OK' + set +o xtrace [2026-07-17T02:40:33+0000] compare_kubectl: statefulset/some-name-cfg OK + desc 'test 1.23.0' + set +o xtrace ----------------------------------------------------------------------------------- test 1.23.0 ----------------------------------------------------------------------------------- + kubectl_bin patch psmdb some-name --type=merge --patch $'{\n "spec": {"crVersion":"1.23.0"}\n }' ++ mktemp + local LAST_OUT=/tmp/tmp.Z91L601ZYi ++ mktemp + local LAST_ERR=/tmp/tmp.ODBOCZTqac + local exit_status=0 + local timeout=4 + args=('patch' 'psmdb' 'some-name' '--type=merge' '--patch' '{ "spec": {"crVersion":"1.23.0"} }') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name --type=merge --patch $'{\n "spec": {"crVersion":"1.23.0"}\n }' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Z91L601ZYi perconaservermongodb.psmdb.percona.com/some-name patched + cat /tmp/tmp.ODBOCZTqac + rm /tmp/tmp.Z91L601ZYi /tmp/tmp.ODBOCZTqac + return 0 + sleep 20 + desc 'check if Pod started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod started ----------------------------------------------------------------------------------- + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PeXD9I37zi +++ mktemp ++ local LAST_ERR=/tmp/tmp.5tsZeOWqBk ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PeXD9I37zi ++ cat /tmp/tmp.5tsZeOWqBk ++ rm /tmp/tmp.PeXD9I37zi /tmp/tmp.5tsZeOWqBk ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MbHMM2vOUk +++ mktemp ++ local LAST_ERR=/tmp/tmp.RFVYaRfyiZ ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MbHMM2vOUk ++ cat /tmp/tmp.RFVYaRfyiZ ++ rm /tmp/tmp.MbHMM2vOUk /tmp/tmp.RFVYaRfyiZ ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aP0iYlWuE5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.e7p1oq9Kr9 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aP0iYlWuE5 ++ cat /tmp/tmp.e7p1oq9Kr9 ++ rm /tmp/tmp.aP0iYlWuE5 /tmp/tmp.e7p1oq9Kr9 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready............................................................................................................................................................................... + wait_for_running some-name-cfg 3 + local name=some-name-cfg + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vLm3JXSdkR +++ mktemp ++ local LAST_ERR=/tmp/tmp.5vq0dmy0um ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vLm3JXSdkR ++ cat /tmp/tmp.5vq0dmy0um ++ rm /tmp/tmp.vLm3JXSdkR /tmp/tmp.5vq0dmy0um ++ return 0 + [[ '' == true ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fzAbbHkBCt +++ mktemp ++ local LAST_ERR=/tmp/tmp.j5uD9aKzs2 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fzAbbHkBCt ++ cat /tmp/tmp.j5uD9aKzs2 ++ rm /tmp/tmp.fzAbbHkBCt /tmp/tmp.j5uD9aKzs2 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cgiEhqfe04 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ki3QR8uMxn ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cgiEhqfe04 ++ cat /tmp/tmp.Ki3QR8uMxn ++ rm /tmp/tmp.cgiEhqfe04 /tmp/tmp.Ki3QR8uMxn ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C343vrvT2u +++ mktemp ++ local LAST_ERR=/tmp/tmp.5T7RVfeVdc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C343vrvT2u ++ cat /tmp/tmp.5T7RVfeVdc ++ rm /tmp/tmp.C343vrvT2u /tmp/tmp.5T7RVfeVdc ++ return 0 + [[ '' == true ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rdehuCijDI +++ mktemp ++ local LAST_ERR=/tmp/tmp.HXjdf29YxR ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rdehuCijDI ++ cat /tmp/tmp.HXjdf29YxR ++ rm /tmp/tmp.rdehuCijDI /tmp/tmp.HXjdf29YxR ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VNhz05obFU +++ mktemp ++ local LAST_ERR=/tmp/tmp.fNR7nBSo2W ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VNhz05obFU ++ cat /tmp/tmp.fNR7nBSo2W ++ rm /tmp/tmp.VNhz05obFU /tmp/tmp.fNR7nBSo2W ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster some-name to be ready' waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fAeygyT7Kh +++ mktemp ++ local LAST_ERR=/tmp/tmp.7H5NtD3MEK ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fAeygyT7Kh ++ cat /tmp/tmp.7H5NtD3MEK ++ rm /tmp/tmp.fAeygyT7Kh /tmp/tmp.7H5NtD3MEK ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_generation 7 statefulset some-name-rs0 + local generation=7 + local resource_type=statefulset + local resource_name=some-name-rs0 + local current_generation ++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LklXs0cXgp +++ mktemp ++ local LAST_ERR=/tmp/tmp.itoDAfuZTD ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-rs0' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.LklXs0cXgp ++ cat /tmp/tmp.itoDAfuZTD ++ rm /tmp/tmp.LklXs0cXgp /tmp/tmp.itoDAfuZTD ++ return 0 + current_generation=7 + [[ 7 != 7 ]] + compare_generation 7 statefulset some-name-cfg + local generation=7 + local resource_type=statefulset + local resource_name=some-name-cfg + local current_generation ++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gvy6AOga7b +++ mktemp ++ local LAST_ERR=/tmp/tmp.BgdcNethQc ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'statefulset' 'some-name-cfg' '-o' 'jsonpath={.metadata.generation}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gvy6AOga7b ++ cat /tmp/tmp.BgdcNethQc ++ rm /tmp/tmp.gvy6AOga7b /tmp/tmp.BgdcNethQc ++ return 0 + current_generation=7 + [[ 7 != 7 ]] + renew_certificate some-name-ssl + certificate=some-name-ssl + wait_certificate some-name-ssl + certificate=some-name-ssl + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:22.860022 2576931 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256380511313000&timeoutSeconds=317&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:24.624850 2576954 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256380504306000&timeoutSeconds=322&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:26.507443 2576972 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256385511644000&timeoutSeconds=301&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:28.274246 2576996 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256385504317000&timeoutSeconds=532&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:30.040918 2577018 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256385504317000&timeoutSeconds=313&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:31.800614 2577040 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256390511722000&timeoutSeconds=578&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:33.534497 2577063 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256390504366000&timeoutSeconds=449&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:35.300364 2577083 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256390504366000&timeoutSeconds=590&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:37.067485 2577105 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256395504418000&timeoutSeconds=390&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + for i in {1..10} + kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s E0717 02:46:38.828237 2577123 reflector.go:227] "Failed to watch" err="Get \"https://34.27.158.45/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-2846/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1784256395511765000&timeoutSeconds=593&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" certificate.cert-manager.io/some-name-ssl condition met + sleep 1 + desc 'renew some-name-ssl' + set +o xtrace ----------------------------------------------------------------------------------- renew some-name-ssl ----------------------------------------------------------------------------------- + local pod_name ++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fGnIRJiomv +++ mktemp ++ local LAST_ERR=/tmp/tmp.LVikMme81H ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'pods' '--selector=name=cmctl' '-o' 'jsonpath={.items[].metadata.name}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fGnIRJiomv ++ cat /tmp/tmp.LVikMme81H ++ rm /tmp/tmp.fGnIRJiomv /tmp/tmp.LVikMme81H ++ return 0 + pod_name=cmctl-5bdf95f5b4-xm7mp + local revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nTjXv3Z3fO +++ mktemp ++ local LAST_ERR=/tmp/tmp.tPdBIGa2It ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nTjXv3Z3fO ++ cat /tmp/tmp.tPdBIGa2It ++ rm /tmp/tmp.nTjXv3Z3fO /tmp/tmp.tPdBIGa2It ++ return 0 + revision=3 + kubectl_bin exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl ++ mktemp + local LAST_OUT=/tmp/tmp.trgQLBr7qT ++ mktemp + local LAST_ERR=/tmp/tmp.7YQhMwqMoJ + local exit_status=0 + local timeout=4 + args=('exec' 'cmctl-5bdf95f5b4-xm7mp' '--' '/tmp/cmctl' 'renew' 'some-name-ssl') + local args + transformed_files=() + local transformed_files + arch_scheduling_enabled + [[ -n '' ]] + arch_scheduling_enabled + [[ -n '' ]] ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec cmctl-5bdf95f5b4-xm7mp -- /tmp/cmctl renew some-name-ssl + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.trgQLBr7qT Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-2846/some-name-ssl + cat /tmp/tmp.7YQhMwqMoJ + rm /tmp/tmp.trgQLBr7qT /tmp/tmp.7YQhMwqMoJ + return 0 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NmdefoOpRW +++ mktemp ++ local LAST_ERR=/tmp/tmp.hskckfIwGH ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.NmdefoOpRW ++ cat /tmp/tmp.hskckfIwGH ++ rm /tmp/tmp.NmdefoOpRW /tmp/tmp.hskckfIwGH ++ return 0 + new_revision=3 + '[' 4 == 3 ']' + sleep 1 + for i in {1..10} + local new_revision ++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e8b2jjf1Yv +++ mktemp ++ local LAST_ERR=/tmp/tmp.aEbw1lNdTy ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'certificate' 'some-name-ssl' '-o' 'jsonpath={.status.revision}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.e8b2jjf1Yv ++ cat /tmp/tmp.aEbw1lNdTy ++ rm /tmp/tmp.e8b2jjf1Yv /tmp/tmp.aEbw1lNdTy ++ return 0 + new_revision=4 + '[' 4 == 4 ']' + break + sleep 20 + wait_cluster + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readiness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0EPdwuVmij +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ief5HspjDj ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0EPdwuVmij ++ cat /tmp/tmp.Ief5HspjDj ++ rm /tmp/tmp.0EPdwuVmij /tmp/tmp.Ief5HspjDj ++ return 0 + [[ '' == true ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WnoiQacIZx +++ mktemp ++ local LAST_ERR=/tmp/tmp.cNvSLQO5Ey ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WnoiQacIZx ++ cat /tmp/tmp.cNvSLQO5Ey ++ rm /tmp/tmp.WnoiQacIZx /tmp/tmp.cNvSLQO5Ey ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k5UnDvNZgY +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZFOeWUfkp5 ++ local exit_status=0 ++ local timeout=4 ++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}') ++ local args ++ transformed_files=() ++ local transformed_files ++ arch_scheduling_enabled ++ [[ -n '' ]] ++ arch_scheduling_enabled ++ [[ -n '' ]] +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.k5UnDvNZgY ++ cat /tmp/tmp.ZFOeWUfkp5 ++ rm /tmp/tmp.k5UnDvNZgY /tmp/tmp.ZFOeWUfkp5 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster some-name to be ready........................................................................................................................