Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/upgrade-consistency-sharded-tls.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ CLUSTER=some-name
+ main
+ create_infra upgrade-consistency-sharded-tls-5481
+ local ns=upgrade-consistency-sharded-tls-5481
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.nOuEZAUuk0
++ mktemp
+ local LAST_ERR=/tmp/tmp.cTGMjkmqgD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nOuEZAUuk0
+ cat /tmp/tmp.cTGMjkmqgD
+ rm /tmp/tmp.nOuEZAUuk0 /tmp/tmp.cTGMjkmqgD
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.evR6eoiaMm
++ mktemp
+ local LAST_ERR=/tmp/tmp.okne55cfwF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.evR6eoiaMm
+ cat /tmp/tmp.okne55cfwF
+ rm /tmp/tmp.evR6eoiaMm /tmp/tmp.okne55cfwF
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nd2SzgP4O6
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZDCSaqUDky
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nd2SzgP4O6
+ cat /tmp/tmp.ZDCSaqUDky
+ rm /tmp/tmp.nd2SzgP4O6 /tmp/tmp.ZDCSaqUDky
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.1eO8HQvzQh
++ mktemp
+ local LAST_ERR=/tmp/tmp.57OFeozjSU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1eO8HQvzQh
+ cat /tmp/tmp.57OFeozjSU
+ rm /tmp/tmp.1eO8HQvzQh /tmp/tmp.57OFeozjSU
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ENBxnwR3Hc
++ mktemp
+ local LAST_ERR=/tmp/tmp.2FnTXgk69C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ENBxnwR3Hc
+ cat /tmp/tmp.2FnTXgk69C
+ rm /tmp/tmp.ENBxnwR3Hc /tmp/tmp.2FnTXgk69C
+ return 0
+ check_crd_for_deletion PR-1961-db56ee7c
+ local git_tag=PR-1961-db56ee7c
++ /usr/bin/sed s/---//g
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-db56ee7c/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.isQMi2ZMYI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W3gLPE7FFq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.isQMi2ZMYI
++ cat /tmp/tmp.W3gLPE7FFq
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.isQMi2ZMYI
++ cat /tmp/tmp.W3gLPE7FFq
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.isQMi2ZMYI
++ cat /tmp/tmp.W3gLPE7FFq
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.isQMi2ZMYI
++ cat /tmp/tmp.W3gLPE7FFq
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.isQMi2ZMYI /tmp/tmp.W3gLPE7FFq
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
+ xargs kubectl delete ns
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wq212VpKsq
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.DhVc9d96CK
++ mktemp
+ local LAST_ERR=/tmp/tmp.XMuqao1ifZ
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.H8Q2KYLeD9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DhVc9d96CK
+ cat /tmp/tmp.H8Q2KYLeD9
+ rm /tmp/tmp.DhVc9d96CK /tmp/tmp.H8Q2KYLeD9
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Wq212VpKsq
+ cat /tmp/tmp.XMuqao1ifZ
+ rm /tmp/tmp.Wq212VpKsq /tmp/tmp.XMuqao1ifZ
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wU8DawRUQe
++ mktemp
+ local LAST_ERR=/tmp/tmp.e0tTWTvPhG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wU8DawRUQe
+ cat /tmp/tmp.e0tTWTvPhG
+ rm /tmp/tmp.wU8DawRUQe /tmp/tmp.e0tTWTvPhG
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
namespace "gmp-system" deleted
+ local LAST_OUT=/tmp/tmp.nYNoNcLxZi
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZSmUz5mAvo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nYNoNcLxZi
namespace/psmdb-operator created
+ cat /tmp/tmp.ZSmUz5mAvo
+ rm /tmp/tmp.nYNoNcLxZi /tmp/tmp.ZSmUz5mAvo
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZIj4pN7Goa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d9WjYkX2D7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZIj4pN7Goa
++ cat /tmp/tmp.d9WjYkX2D7
++ rm /tmp/tmp.ZIj4pN7Goa /tmp/tmp.d9WjYkX2D7
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.G5fHvaWpUA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ai0iBUCgXA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G5fHvaWpUA
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6" modified.
+ cat /tmp/tmp.Ai0iBUCgXA
+ rm /tmp/tmp.G5fHvaWpUA /tmp/tmp.Ai0iBUCgXA
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.MgxrtqqThz
++ mktemp
+ local LAST_ERR=/tmp/tmp.wvDgBq6VQ8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MgxrtqqThz
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.wvDgBq6VQ8
+ rm /tmp/tmp.MgxrtqqThz /tmp/tmp.wvDgBq6VQ8
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BAXKrI5I2D
++ mktemp
+ local LAST_ERR=/tmp/tmp.RbxH7YB5pS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BAXKrI5I2D
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.RbxH7YB5pS
+ rm /tmp/tmp.BAXKrI5I2D /tmp/tmp.RbxH7YB5pS
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-db56ee7c") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.40uwTQTeMR
++ mktemp
+ local LAST_ERR=/tmp/tmp.np3WzLY3N5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.40uwTQTeMR
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.np3WzLY3N5
+ rm /tmp/tmp.40uwTQTeMR /tmp/tmp.np3WzLY3N5
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fCToUSoybn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A9QiIeMu3u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fCToUSoybn
++ cat /tmp/tmp.A9QiIeMu3u
++ rm /tmp/tmp.fCToUSoybn /tmp/tmp.A9QiIeMu3u
++ return 0
+ wait_pod percona-server-mongodb-operator-d49c7dc6d-bdkfl
+ local pod=percona-server-mongodb-operator-d49c7dc6d-bdkfl
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-d49c7dc6d-bdkfl to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MR2H3qLwV1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EnGd3mvUBD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MR2H3qLwV1
++ cat /tmp/tmp.EnGd3mvUBD
++ rm /tmp/tmp.MR2H3qLwV1 /tmp/tmp.EnGd3mvUBD
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-d49c7dc6d-bdkfl
++ mktemp
+ local LAST_OUT=/tmp/tmp.5cXlXYN3Ce
++ mktemp
+ local LAST_ERR=/tmp/tmp.18o22w91m5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-d49c7dc6d-bdkfl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5cXlXYN3Ce
+ cat /tmp/tmp.18o22w91m5
+ rm /tmp/tmp.5cXlXYN3Ce /tmp/tmp.18o22w91m5
+ return 0
2025-06-06T08:18:04.674Z	INFO	setup	Manager starting up	{"gitCommit": "db56ee7c993f5ad2d1ada9b0fbbd4e3142f61963", "gitBranch": "PR-1961-db56ee7c", "buildTime": "", "goVersion": "go1.24.4", "os": "linux", "arch": "amd64"}
+ create_namespace upgrade-consistency-sharded-tls-5481
+ local namespace=upgrade-consistency-sharded-tls-5481
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-5481'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-sharded-tls-5481
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-sharded-tls-5481 --ignore-not-found
+ kubectl_bin get ns
+ xargs kubectl delete ns
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vRZvZ47Se1
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.yxi5vhgKhX
++ mktemp
+ local LAST_ERR=/tmp/tmp.ARPfUI8jop
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.tE88cBM97s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-sharded-tls-5481 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vRZvZ47Se1
+ cat /tmp/tmp.ARPfUI8jop
+ rm /tmp/tmp.vRZvZ47Se1 /tmp/tmp.ARPfUI8jop
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yxi5vhgKhX
+ cat /tmp/tmp.tE88cBM97s
+ rm /tmp/tmp.yxi5vhgKhX /tmp/tmp.tE88cBM97s
+ return 0
+ kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-5481
++ mktemp
+ local LAST_OUT=/tmp/tmp.6BeAowD1Gh
++ mktemp
+ local LAST_ERR=/tmp/tmp.eKpSkV96bM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-5481
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6BeAowD1Gh
+ cat /tmp/tmp.eKpSkV96bM
+ rm /tmp/tmp.6BeAowD1Gh /tmp/tmp.eKpSkV96bM
+ return 0
+ desc 'create namespace upgrade-consistency-sharded-tls-5481'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-sharded-tls-5481
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-sharded-tls-5481
++ mktemp
+ local LAST_OUT=/tmp/tmp.5Jy3ZINKq6
namespace "gmp-public" deleted
++ mktemp
+ local LAST_ERR=/tmp/tmp.59DI33RZ9G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-consistency-sharded-tls-5481
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5Jy3ZINKq6
namespace/upgrade-consistency-sharded-tls-5481 created
+ cat /tmp/tmp.59DI33RZ9G
+ rm /tmp/tmp.5Jy3ZINKq6 /tmp/tmp.59DI33RZ9G
+ return 0
+ set_kube_ctx upgrade-consistency-sharded-tls-5481
+ local namespace=upgrade-consistency-sharded-tls-5481
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2kS0H0Nte5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xSfDAnAGRR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2kS0H0Nte5
++ cat /tmp/tmp.xSfDAnAGRR
++ rm /tmp/tmp.2kS0H0Nte5 /tmp/tmp.xSfDAnAGRR
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6 --namespace=upgrade-consistency-sharded-tls-5481
++ mktemp
+ local LAST_OUT=/tmp/tmp.td8yL9jVfA
++ mktemp
+ local LAST_ERR=/tmp/tmp.KCcY0f5Szs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6 --namespace=upgrade-consistency-sharded-tls-5481
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.td8yL9jVfA
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-db56ee7c-3-cluster6" modified.
+ cat /tmp/tmp.KCcY0f5Szs
+ rm /tmp/tmp.td8yL9jVfA /tmp/tmp.KCcY0f5Szs
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.1jXnKtVsAZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.hEzNMmNHN5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1jXnKtVsAZ
namespace/cert-manager created
+ cat /tmp/tmp.hEzNMmNHN5
+ rm /tmp/tmp.1jXnKtVsAZ /tmp/tmp.hEzNMmNHN5
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.4PnP6Fqy9m
++ mktemp
+ local LAST_ERR=/tmp/tmp.FAwATZJvs0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4PnP6Fqy9m
namespace/cert-manager labeled
+ cat /tmp/tmp.FAwATZJvs0
+ rm /tmp/tmp.4PnP6Fqy9m /tmp/tmp.FAwATZJvs0
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gheq4Zdb0o
++ mktemp
+ local LAST_ERR=/tmp/tmp.cyNbiPMG1x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gheq4Zdb0o
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created
clusterrole.rbac.authorization.k8s.io/cert-manager-view created
clusterrole.rbac.authorization.k8s.io/cert-manager-edit created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created
role.rbac.authorization.k8s.io/cert-manager:leaderelection created
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created
rolebinding.rbac.authorization.k8s.io/cert-manager-cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created
+ cat /tmp/tmp.cyNbiPMG1x
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.Gheq4Zdb0o /tmp/tmp.cyNbiPMG1x
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.r1p8vXqDA6
++ mktemp
+ local LAST_ERR=/tmp/tmp.aWFCNxPeOy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r1p8vXqDA6
pod/cert-manager-6687d8765c-56mkm condition met
pod/cert-manager-cainjector-764498cfc8-xzt9v condition met
pod/cert-manager-webhook-74c74b87d7-nxd26 condition met
+ cat /tmp/tmp.aWFCNxPeOy
+ rm /tmp/tmp.r1p8vXqDA6 /tmp/tmp.aWFCNxPeOy
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.CEAgLvLDih
++ mktemp
+ local LAST_ERR=/tmp/tmp.TlsyBvaXP9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CEAgLvLDih
secret/some-users created
+ cat /tmp/tmp.TlsyBvaXP9
+ rm /tmp/tmp.CEAgLvLDih /tmp/tmp.TlsyBvaXP9
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ENr4SvbdE0
++ mktemp
+ local LAST_ERR=/tmp/tmp.IlQTvzOW6d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ENr4SvbdE0
deployment.apps/psmdb-client created
+ cat /tmp/tmp.IlQTvzOW6d
+ rm /tmp/tmp.ENr4SvbdE0 /tmp/tmp.IlQTvzOW6d
+ return 0
+ deploy_cmctl
+ local service_account=cmctl
+ /usr/bin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/rbac.yaml
+ yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CgChmC7QnS
++ mktemp
+ local LAST_ERR=/tmp/tmp.puKlKfnn3t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CgChmC7QnS
role.rbac.authorization.k8s.io/cmctl created
serviceaccount/cmctl created
rolebinding.rbac.authorization.k8s.io/service-account-cmctl created
+ cat /tmp/tmp.puKlKfnn3t
+ rm /tmp/tmp.CgChmC7QnS /tmp/tmp.puKlKfnn3t
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cmctl.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.k8jNNfsSDm
++ mktemp
+ local LAST_ERR=/tmp/tmp.rHPL6IdFZR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cmctl.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k8jNNfsSDm
deployment.apps/cmctl created
+ cat /tmp/tmp.rHPL6IdFZR
+ rm /tmp/tmp.k8jNNfsSDm /tmp/tmp.rHPL6IdFZR
+ return 0
+ desc 'create first PSMDB cluster 1.19.1 some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster 1.19.1 some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-db56ee7c"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.DPa3dXVyMH
++ mktemp
+ local LAST_ERR=/tmp/tmp.q1T27nD99Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DPa3dXVyMH
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.q1T27nD99Z
+ rm /tmp/tmp.DPa3dXVyMH /tmp/tmp.q1T27nD99Z
+ return 0
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8aX9vKjesL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J9usbPiFt5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8aX9vKjesL
++ cat /tmp/tmp.J9usbPiFt5
++ rm /tmp/tmp.8aX9vKjesL /tmp/tmp.J9usbPiFt5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready...........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.58VstFzfVv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KpljPA9CKH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.58VstFzfVv
++ cat /tmp/tmp.KpljPA9CKH
++ rm /tmp/tmp.58VstFzfVv /tmp/tmp.KpljPA9CKH
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yTZw7VTj4J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GHN9KFO6s9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yTZw7VTj4J
++ cat /tmp/tmp.GHN9KFO6s9
++ rm /tmp/tmp.yTZw7VTj4J /tmp/tmp.GHN9KFO6s9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qeDhusVBNO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ipuIQBbZy2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qeDhusVBNO
++ cat /tmp/tmp.ipuIQBbZy2
++ rm /tmp/tmp.qeDhusVBNO /tmp/tmp.ipuIQBbZy2
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pfjXdw00vv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AR5t3doOfZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pfjXdw00vv
++ cat /tmp/tmp.AR5t3doOfZ
++ rm /tmp/tmp.pfjXdw00vv /tmp/tmp.AR5t3doOfZ
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U1ScnFIFUy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eW7z5OV6Uj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U1ScnFIFUy
++ cat /tmp/tmp.eW7z5OV6Uj
++ rm /tmp/tmp.U1ScnFIFUy /tmp/tmp.eW7z5OV6Uj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bUKTEaI0vc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yh1IZUlnyS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bUKTEaI0vc
++ cat /tmp/tmp.Yh1IZUlnyS
++ rm /tmp/tmp.bUKTEaI0vc /tmp/tmp.Yh1IZUlnyS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PTxv6sI5VJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KidVsOsJvJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PTxv6sI5VJ
++ cat /tmp/tmp.KidVsOsJvJ
++ rm /tmp/tmp.PTxv6sI5VJ /tmp/tmp.KidVsOsJvJ
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JSaIEgENsE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Y8FY3O45N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JSaIEgENsE
++ cat /tmp/tmp.7Y8FY3O45N
++ rm /tmp/tmp.JSaIEgENsE /tmp/tmp.7Y8FY3O45N
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NrViERquUY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2jzOtrpqXt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NrViERquUY
++ cat /tmp/tmp.2jzOtrpqXt
++ rm /tmp/tmp.NrViERquUY /tmp/tmp.2jzOtrpqXt
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 1 statefulset some-name-rs0
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qdl9SpR5Ie
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yjRhW5OBK7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Qdl9SpR5Ie
++ cat /tmp/tmp.yjRhW5OBK7
++ rm /tmp/tmp.Qdl9SpR5Ie /tmp/tmp.yjRhW5OBK7
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ compare_generation 1 statefulset some-name-cfg
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t8kSofJdaj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q23rq0ihiE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t8kSofJdaj
++ cat /tmp/tmp.Q23rq0ihiE
++ rm /tmp/tmp.t8kSofJdaj /tmp/tmp.Q23rq0ihiE
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hjBRBSrsDx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ULU23V1mZr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hjBRBSrsDx
++ cat /tmp/tmp.ULU23V1mZr
++ rm /tmp/tmp.hjBRBSrsDx /tmp/tmp.ULU23V1mZr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iV2rBKbIne
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MGLSnpieJt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iV2rBKbIne
++ cat /tmp/tmp.MGLSnpieJt
++ rm /tmp/tmp.iV2rBKbIne /tmp/tmp.MGLSnpieJt
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EpE6Xl941n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t8wsTKgzox
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EpE6Xl941n
++ cat /tmp/tmp.t8wsTKgzox
++ rm /tmp/tmp.EpE6Xl941n /tmp/tmp.t8wsTKgzox
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TmyeN3KEG9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4a85vdiK37
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TmyeN3KEG9
++ cat /tmp/tmp.4a85vdiK37
++ rm /tmp/tmp.TmyeN3KEG9 /tmp/tmp.4a85vdiK37
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PkSS0GHxET
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nzf2DPHWwz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PkSS0GHxET
++ cat /tmp/tmp.Nzf2DPHWwz
++ rm /tmp/tmp.PkSS0GHxET /tmp/tmp.Nzf2DPHWwz
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hilj07WGoK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.L6QHz5exBg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Hilj07WGoK
++ cat /tmp/tmp.L6QHz5exBg
++ rm /tmp/tmp.Hilj07WGoK /tmp/tmp.L6QHz5exBg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GJVHsmzode
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2sX6PL7aIB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GJVHsmzode
++ cat /tmp/tmp.2sX6PL7aIB
++ rm /tmp/tmp.GJVHsmzode /tmp/tmp.2sX6PL7aIB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RAhl7ibyeK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VEyq7CtU70
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RAhl7ibyeK
++ cat /tmp/tmp.VEyq7CtU70
++ rm /tmp/tmp.RAhl7ibyeK /tmp/tmp.VEyq7CtU70
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4DKFalmZzb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nzR8IzlkBA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4DKFalmZzb
++ cat /tmp/tmp.nzR8IzlkBA
++ rm /tmp/tmp.4DKFalmZzb /tmp/tmp.nzR8IzlkBA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A2fXQAc0LQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GCzFHyEqRs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A2fXQAc0LQ
++ cat /tmp/tmp.GCzFHyEqRs
++ rm /tmp/tmp.A2fXQAc0LQ /tmp/tmp.GCzFHyEqRs
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oZaQsIgXLk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vsfoxPYrQF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oZaQsIgXLk
++ cat /tmp/tmp.vsfoxPYrQF
++ rm /tmp/tmp.oZaQsIgXLk /tmp/tmp.vsfoxPYrQF
++ return 0
+ pod_name=cmctl-59494fbfb5-t2qdf
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WTRf73G9di
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I9ZbriyFUx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WTRf73G9di
++ cat /tmp/tmp.I9ZbriyFUx
++ rm /tmp/tmp.WTRf73G9di /tmp/tmp.I9ZbriyFUx
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.4qCCBB7tts
++ mktemp
+ local LAST_ERR=/tmp/tmp.oWHjSuJBj7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4qCCBB7tts
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5481/some-name-ssl
+ cat /tmp/tmp.oWHjSuJBj7
+ rm /tmp/tmp.4qCCBB7tts /tmp/tmp.oWHjSuJBj7
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5uPj8zvXqP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3j3DVmvmvU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5uPj8zvXqP
++ cat /tmp/tmp.3j3DVmvmvU
++ rm /tmp/tmp.5uPj8zvXqP /tmp/tmp.3j3DVmvmvU
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BJcceOJoBW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fGYV3duch7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BJcceOJoBW
++ cat /tmp/tmp.fGYV3duch7
++ rm /tmp/tmp.BJcceOJoBW /tmp/tmp.fGYV3duch7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BP8QiK9cdX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ECoOQ80DoL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BP8QiK9cdX
++ cat /tmp/tmp.ECoOQ80DoL
++ rm /tmp/tmp.BP8QiK9cdX /tmp/tmp.ECoOQ80DoL
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iquciMZNSc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ish7LsAPOr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iquciMZNSc
++ cat /tmp/tmp.Ish7LsAPOr
++ rm /tmp/tmp.iquciMZNSc /tmp/tmp.Ish7LsAPOr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.......................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pPWn0kqRQD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nW1vrQZgD6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pPWn0kqRQD
++ cat /tmp/tmp.nW1vrQZgD6
++ rm /tmp/tmp.pPWn0kqRQD /tmp/tmp.nW1vrQZgD6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V3i3YqM6T9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0RDUQuw3v5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V3i3YqM6T9
++ cat /tmp/tmp.0RDUQuw3v5
++ rm /tmp/tmp.V3i3YqM6T9 /tmp/tmp.0RDUQuw3v5
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gfP1dOdymH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YzWu0myVxo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gfP1dOdymH
++ cat /tmp/tmp.YzWu0myVxo
++ rm /tmp/tmp.gfP1dOdymH /tmp/tmp.YzWu0myVxo
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gAcavc5m9j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PUWWbjMU50
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gAcavc5m9j
++ cat /tmp/tmp.PUWWbjMU50
++ rm /tmp/tmp.gAcavc5m9j /tmp/tmp.PUWWbjMU50
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HQodqjGcvw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IMHQFl5dOA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HQodqjGcvw
++ cat /tmp/tmp.IMHQFl5dOA
++ rm /tmp/tmp.HQodqjGcvw /tmp/tmp.IMHQFl5dOA
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CXeUfHCKKi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.93NCOIGJAZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CXeUfHCKKi
++ cat /tmp/tmp.93NCOIGJAZ
++ rm /tmp/tmp.CXeUfHCKKi /tmp/tmp.93NCOIGJAZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DwEciGaY9y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pCwf5HYq0K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DwEciGaY9y
++ cat /tmp/tmp.pCwf5HYq0K
++ rm /tmp/tmp.DwEciGaY9y /tmp/tmp.pCwf5HYq0K
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 2 statefulset some-name-rs0
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0AYFxqqZyY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uaBxumKSbh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0AYFxqqZyY
++ cat /tmp/tmp.uaBxumKSbh
++ rm /tmp/tmp.0AYFxqqZyY /tmp/tmp.uaBxumKSbh
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ compare_generation 2 statefulset some-name-cfg
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7DMwrVRg23
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LQOKW23Qid
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7DMwrVRg23
++ cat /tmp/tmp.LQOKW23Qid
++ rm /tmp/tmp.7DMwrVRg23 /tmp/tmp.LQOKW23Qid
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uJy9fYhMEV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y5EtyLZoZV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uJy9fYhMEV
++ cat /tmp/tmp.Y5EtyLZoZV
++ rm /tmp/tmp.uJy9fYhMEV /tmp/tmp.Y5EtyLZoZV
++ return 0
+ pod_name=cmctl-59494fbfb5-t2qdf
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cJJqQsZuo5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.orKnCly7sy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cJJqQsZuo5
++ cat /tmp/tmp.orKnCly7sy
++ rm /tmp/tmp.cJJqQsZuo5 /tmp/tmp.orKnCly7sy
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.jdoofrcFoy
++ mktemp
+ local LAST_ERR=/tmp/tmp.10CKhxZUmR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jdoofrcFoy
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5481/some-name-ssl-internal
+ cat /tmp/tmp.10CKhxZUmR
+ rm /tmp/tmp.jdoofrcFoy /tmp/tmp.10CKhxZUmR
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M962ImlO9Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RHjub88e7k
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M962ImlO9Y
++ cat /tmp/tmp.RHjub88e7k
++ rm /tmp/tmp.M962ImlO9Y /tmp/tmp.RHjub88e7k
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E5xbMplDxj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xcxiZc5dbG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E5xbMplDxj
++ cat /tmp/tmp.xcxiZc5dbG
++ rm /tmp/tmp.E5xbMplDxj /tmp/tmp.xcxiZc5dbG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bNRZXIXAm0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r9vsjsn100
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bNRZXIXAm0
++ cat /tmp/tmp.r9vsjsn100
++ rm /tmp/tmp.bNRZXIXAm0 /tmp/tmp.r9vsjsn100
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Wp8mu6iQyC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4PKRMXmKqW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Wp8mu6iQyC
++ cat /tmp/tmp.4PKRMXmKqW
++ rm /tmp/tmp.Wp8mu6iQyC /tmp/tmp.4PKRMXmKqW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness............................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9gvm3s5r94
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YQyNgt3jIz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9gvm3s5r94
++ cat /tmp/tmp.YQyNgt3jIz
++ rm /tmp/tmp.9gvm3s5r94 /tmp/tmp.YQyNgt3jIz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jiOXnHX9Lv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VltfMAy1hS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jiOXnHX9Lv
++ cat /tmp/tmp.VltfMAy1hS
++ rm /tmp/tmp.jiOXnHX9Lv /tmp/tmp.VltfMAy1hS
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vzi3aBKhlM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZewUygpVRk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vzi3aBKhlM
++ cat /tmp/tmp.ZewUygpVRk
++ rm /tmp/tmp.vzi3aBKhlM /tmp/tmp.ZewUygpVRk
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1KgJXct52S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bGQ4h96xVt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1KgJXct52S
++ cat /tmp/tmp.bGQ4h96xVt
++ rm /tmp/tmp.1KgJXct52S /tmp/tmp.bGQ4h96xVt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.33vaCNRQCb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4wmxrV8g0c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.33vaCNRQCb
++ cat /tmp/tmp.4wmxrV8g0c
++ rm /tmp/tmp.33vaCNRQCb /tmp/tmp.4wmxrV8g0c
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ytJUWnSd1M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H4bkxplZTR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ytJUWnSd1M
++ cat /tmp/tmp.H4bkxplZTR
++ rm /tmp/tmp.ytJUWnSd1M /tmp/tmp.H4bkxplZTR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8F41zZ9NDr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gKpFPbJY1l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8F41zZ9NDr
++ cat /tmp/tmp.gKpFPbJY1l
++ rm /tmp/tmp.8F41zZ9NDr /tmp/tmp.gKpFPbJY1l
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P0JItAEI2G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FbTkHw7bqh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P0JItAEI2G
++ cat /tmp/tmp.FbTkHw7bqh
++ rm /tmp/tmp.P0JItAEI2G /tmp/tmp.FbTkHw7bqh
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9BtHyzpDzP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IUdURuIYCJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9BtHyzpDzP
++ cat /tmp/tmp.IUdURuIYCJ
++ rm /tmp/tmp.9BtHyzpDzP /tmp/tmp.IUdURuIYCJ
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1191
+ local resource=service/some-name-rs0
+ local postfix=-1191
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1191.yml
+ local new_result=/tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1191-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6WbjtoPVT6
++ mktemp
+ local LAST_ERR=/tmp/tmp.iXoTWbPXGW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6WbjtoPVT6
+ cat /tmp/tmp.iXoTWbPXGW
+ rm /tmp/tmp.6WbjtoPVT6 /tmp/tmp.iXoTWbPXGW
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.30 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1191.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1191.yml /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1191
+ local resource=service/some-name-cfg
+ local postfix=-1191
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1191.yml
+ local new_result=/tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1191-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.v5iuH2OWHj
++ mktemp
+ local LAST_ERR=/tmp/tmp.dxSnSO02Fz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v5iuH2OWHj
+ cat /tmp/tmp.dxSnSO02Fz
+ rm /tmp/tmp.v5iuH2OWHj /tmp/tmp.dxSnSO02Fz
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ version_gt 1.22
++ bc -l
++ echo '1.30 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1191.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1191.yml /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1191
+ local resource=statefulset/some-name-rs0
+ local postfix=-1191
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1191.yml
+ local new_result=/tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1191-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3hXYK1CawK
++ mktemp
+ local LAST_ERR=/tmp/tmp.1nxBFdcAs6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3hXYK1CawK
+ cat /tmp/tmp.1nxBFdcAs6
+ rm /tmp/tmp.3hXYK1CawK /tmp/tmp.1nxBFdcAs6
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1191.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1191.yml /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1191
+ local resource=statefulset/some-name-cfg
+ local postfix=-1191
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1191.yml
+ local new_result=/tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1191-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.F1cLSZLK9j
++ mktemp
+ local LAST_ERR=/tmp/tmp.OyB1ml4epY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F1cLSZLK9j
+ cat /tmp/tmp.OyB1ml4epY
+ rm /tmp/tmp.F1cLSZLK9j /tmp/tmp.OyB1ml4epY
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1191.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1191.yml /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ desc 'test 1.20.1'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.20.1
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.20.1"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z8xHLAFkRq
++ mktemp
+ local LAST_ERR=/tmp/tmp.HWbxkKNIJq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.20.1"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z8xHLAFkRq
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.HWbxkKNIJq
+ rm /tmp/tmp.Z8xHLAFkRq /tmp/tmp.HWbxkKNIJq
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YXqbhdrYwb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.30PFtR9AYT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YXqbhdrYwb
++ cat /tmp/tmp.30PFtR9AYT
++ rm /tmp/tmp.YXqbhdrYwb /tmp/tmp.30PFtR9AYT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7ZblA1Z6nw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PDX6lZBE0u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7ZblA1Z6nw
++ cat /tmp/tmp.PDX6lZBE0u
++ rm /tmp/tmp.7ZblA1Z6nw /tmp/tmp.PDX6lZBE0u
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5jzeasNp8p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.277PFWDmc4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5jzeasNp8p
++ cat /tmp/tmp.277PFWDmc4
++ rm /tmp/tmp.5jzeasNp8p /tmp/tmp.277PFWDmc4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ua7s2JdZlw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W0RMZd4bYp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ua7s2JdZlw
++ cat /tmp/tmp.W0RMZd4bYp
++ rm /tmp/tmp.Ua7s2JdZlw /tmp/tmp.W0RMZd4bYp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qnO87ShVcc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O6BhVxfPEL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qnO87ShVcc
++ cat /tmp/tmp.O6BhVxfPEL
++ rm /tmp/tmp.qnO87ShVcc /tmp/tmp.O6BhVxfPEL
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S4r6THya3X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RkGbyE9efY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S4r6THya3X
++ cat /tmp/tmp.RkGbyE9efY
++ rm /tmp/tmp.S4r6THya3X /tmp/tmp.RkGbyE9efY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rxkX8T3F8q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AEbfOTsXo6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rxkX8T3F8q
++ cat /tmp/tmp.AEbfOTsXo6
++ rm /tmp/tmp.rxkX8T3F8q /tmp/tmp.AEbfOTsXo6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XOCD54RxIq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aoz4lDm9gO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XOCD54RxIq
++ cat /tmp/tmp.aoz4lDm9gO
++ rm /tmp/tmp.XOCD54RxIq /tmp/tmp.aoz4lDm9gO
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sH0B4SxBjb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DhI1z7tOjp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sH0B4SxBjb
++ cat /tmp/tmp.DhI1z7tOjp
++ rm /tmp/tmp.sH0B4SxBjb /tmp/tmp.DhI1z7tOjp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9vw6VR2JFh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CWBK2uK9WR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9vw6VR2JFh
++ cat /tmp/tmp.CWBK2uK9WR
++ rm /tmp/tmp.9vw6VR2JFh /tmp/tmp.CWBK2uK9WR
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fCPAXoAZ3a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TvTqfnbIk5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fCPAXoAZ3a
++ cat /tmp/tmp.TvTqfnbIk5
++ rm /tmp/tmp.fCPAXoAZ3a /tmp/tmp.TvTqfnbIk5
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.temRPrywX4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LHmox6Lumg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.temRPrywX4
++ cat /tmp/tmp.LHmox6Lumg
++ rm /tmp/tmp.temRPrywX4 /tmp/tmp.LHmox6Lumg
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g4oZwLwekF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WJbDcZR03U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g4oZwLwekF
++ cat /tmp/tmp.WJbDcZR03U
++ rm /tmp/tmp.g4oZwLwekF /tmp/tmp.WJbDcZR03U
++ return 0
+ pod_name=cmctl-59494fbfb5-t2qdf
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aOYKUSkSOh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UWaE0dsFtG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aOYKUSkSOh
++ cat /tmp/tmp.UWaE0dsFtG
++ rm /tmp/tmp.aOYKUSkSOh /tmp/tmp.UWaE0dsFtG
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.CzKd0v0jQm
++ mktemp
+ local LAST_ERR=/tmp/tmp.uQbuiPbn9a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-59494fbfb5-t2qdf -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CzKd0v0jQm
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5481/some-name-ssl
+ cat /tmp/tmp.uQbuiPbn9a
+ rm /tmp/tmp.CzKd0v0jQm /tmp/tmp.uQbuiPbn9a
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UsZYCkjhDx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xTIyz7rgnx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UsZYCkjhDx
++ cat /tmp/tmp.xTIyz7rgnx
++ rm /tmp/tmp.UsZYCkjhDx /tmp/tmp.xTIyz7rgnx
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.34iuDTUuQF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IsYDFlniK3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.34iuDTUuQF
++ cat /tmp/tmp.IsYDFlniK3
++ rm /tmp/tmp.34iuDTUuQF /tmp/tmp.IsYDFlniK3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hyFhHF9GFh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gMspNmbSOa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hyFhHF9GFh
++ cat /tmp/tmp.gMspNmbSOa
++ rm /tmp/tmp.hyFhHF9GFh /tmp/tmp.gMspNmbSOa
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dfn01mAlzV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SPv7va1zFW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dfn01mAlzV
++ cat /tmp/tmp.SPv7va1zFW
++ rm /tmp/tmp.Dfn01mAlzV /tmp/tmp.SPv7va1zFW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...................................................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9CT2H6ca2H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.27cWDopXNc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9CT2H6ca2H
++ cat /tmp/tmp.27cWDopXNc
++ rm /tmp/tmp.9CT2H6ca2H /tmp/tmp.27cWDopXNc
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pP9GTDypQ8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Am7AfIFwPR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pP9GTDypQ8
++ cat /tmp/tmp.Am7AfIFwPR
++ rm /tmp/tmp.pP9GTDypQ8 /tmp/tmp.Am7AfIFwPR
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T3i48i7MXf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2qQCLeacYG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T3i48i7MXf
++ cat /tmp/tmp.2qQCLeacYG
++ rm /tmp/tmp.T3i48i7MXf /tmp/tmp.2qQCLeacYG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WZbRTGmsVN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fduj8Rfq3f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WZbRTGmsVN
++ cat /tmp/tmp.Fduj8Rfq3f
++ rm /tmp/tmp.WZbRTGmsVN /tmp/tmp.Fduj8Rfq3f
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bwc9yBKWWd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VEdLzAtHvj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Bwc9yBKWWd
++ cat /tmp/tmp.VEdLzAtHvj
++ rm /tmp/tmp.Bwc9yBKWWd /tmp/tmp.VEdLzAtHvj
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HpOOqvxDOY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lI13mbtFL0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HpOOqvxDOY
++ cat /tmp/tmp.lI13mbtFL0
++ rm /tmp/tmp.HpOOqvxDOY /tmp/tmp.lI13mbtFL0
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iZiACMD2cH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eCsSMG8onW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iZiACMD2cH
++ cat /tmp/tmp.eCsSMG8onW
++ rm /tmp/tmp.iZiACMD2cH /tmp/tmp.eCsSMG8onW
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 4 statefulset some-name-rs0
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5O29YS7184
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vc2Vb9XtNL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5O29YS7184
++ cat /tmp/tmp.vc2Vb9XtNL
++ rm /tmp/tmp.5O29YS7184 /tmp/tmp.vc2Vb9XtNL
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ compare_generation 4 statefulset some-name-cfg
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n85Nkrd919
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zpt8wuk7U3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n85Nkrd919
++ cat /tmp/tmp.Zpt8wuk7U3
++ rm /tmp/tmp.n85Nkrd919 /tmp/tmp.Zpt8wuk7U3
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NVafTDTrwI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3iir9Gckcu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NVafTDTrwI
++ cat /tmp/tmp.3iir9Gckcu
++ rm /tmp/tmp.NVafTDTrwI /tmp/tmp.3iir9Gckcu
++ return 0
+ pod_name=cmctl-59494fbfb5-s672m
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ei3eC1FNs4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YuvFFV497f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ei3eC1FNs4
++ cat /tmp/tmp.YuvFFV497f
++ rm /tmp/tmp.ei3eC1FNs4 /tmp/tmp.YuvFFV497f
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-59494fbfb5-s672m -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.XCs9yLYm6F
++ mktemp
+ local LAST_ERR=/tmp/tmp.hjEcfPwznI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-59494fbfb5-s672m -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XCs9yLYm6F
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5481/some-name-ssl-internal
+ cat /tmp/tmp.hjEcfPwznI
+ rm /tmp/tmp.XCs9yLYm6F /tmp/tmp.hjEcfPwznI
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JhCd8q7slF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eKaxP4Azzn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JhCd8q7slF
++ cat /tmp/tmp.eKaxP4Azzn
++ rm /tmp/tmp.JhCd8q7slF /tmp/tmp.eKaxP4Azzn
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yw35NAuzk7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EiLi0UOSCF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Yw35NAuzk7
++ cat /tmp/tmp.EiLi0UOSCF
++ rm /tmp/tmp.Yw35NAuzk7 /tmp/tmp.EiLi0UOSCF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RgTFajQ1Tw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fn6dEmF1de
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RgTFajQ1Tw
++ cat /tmp/tmp.Fn6dEmF1de
++ rm /tmp/tmp.RgTFajQ1Tw /tmp/tmp.Fn6dEmF1de
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xjz8NbmfYa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HyuflUPMnb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xjz8NbmfYa
++ cat /tmp/tmp.HyuflUPMnb
++ rm /tmp/tmp.xjz8NbmfYa /tmp/tmp.HyuflUPMnb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.....................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Pp3XMJYwE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7OCXXeRH1K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2Pp3XMJYwE
++ cat /tmp/tmp.7OCXXeRH1K
++ rm /tmp/tmp.2Pp3XMJYwE /tmp/tmp.7OCXXeRH1K
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.avedXkl1Vu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.plEnSPi1nB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.avedXkl1Vu
++ cat /tmp/tmp.plEnSPi1nB
++ rm /tmp/tmp.avedXkl1Vu /tmp/tmp.plEnSPi1nB
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0H0phoR5Wo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zFLUUmiegv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0H0phoR5Wo
++ cat /tmp/tmp.zFLUUmiegv
++ rm /tmp/tmp.0H0phoR5Wo /tmp/tmp.zFLUUmiegv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uKUJofu2mJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cSO7g2gF6O
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uKUJofu2mJ
++ cat /tmp/tmp.cSO7g2gF6O
++ rm /tmp/tmp.uKUJofu2mJ /tmp/tmp.cSO7g2gF6O
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4UqLDaRbFx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nz9XfHA1ZJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4UqLDaRbFx
++ cat /tmp/tmp.Nz9XfHA1ZJ
++ rm /tmp/tmp.4UqLDaRbFx /tmp/tmp.Nz9XfHA1ZJ
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SSLoELSMwX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eKUidjogaB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SSLoELSMwX
++ cat /tmp/tmp.eKUidjogaB
++ rm /tmp/tmp.SSLoELSMwX /tmp/tmp.eKUidjogaB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HvY7kEA2ry
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7jEauAWfpo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HvY7kEA2ry
++ cat /tmp/tmp.7jEauAWfpo
++ rm /tmp/tmp.HvY7kEA2ry /tmp/tmp.7jEauAWfpo
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_generation 5 statefulset some-name-rs0
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6KJ9Es3n4N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2UTcbJOWs9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6KJ9Es3n4N
++ cat /tmp/tmp.2UTcbJOWs9
++ rm /tmp/tmp.6KJ9Es3n4N /tmp/tmp.2UTcbJOWs9
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ compare_generation 5 statefulset some-name-cfg
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.THNvRHQ4UL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.abGdat3bCZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.THNvRHQ4UL
++ cat /tmp/tmp.abGdat3bCZ
++ rm /tmp/tmp.THNvRHQ4UL /tmp/tmp.abGdat3bCZ
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1201
+ local resource=service/some-name-rs0
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml
+ local new_result=/tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
++ mktemp
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.UowXHm4lZa
++ mktemp
+ local LAST_ERR=/tmp/tmp.E1k7OsGdzi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UowXHm4lZa
+ cat /tmp/tmp.E1k7OsGdzi
+ rm /tmp/tmp.UowXHm4lZa /tmp/tmp.E1k7OsGdzi
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml /tmp/tmp.pAywTItlYw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1201
+ local resource=service/some-name-cfg
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml
+ local new_result=/tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.JavtxOvDvP
++ mktemp
+ local LAST_ERR=/tmp/tmp.MHkcfMlkhN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JavtxOvDvP
+ cat /tmp/tmp.MHkcfMlkhN
+ rm /tmp/tmp.JavtxOvDvP /tmp/tmp.MHkcfMlkhN
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml /tmp/tmp.pAywTItlYw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1201
+ local resource=statefulset/some-name-rs0
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml
+ local new_result=/tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1Njd1oT7Rk
++ mktemp
+ local LAST_ERR=/tmp/tmp.PSrYnBLJmj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1Njd1oT7Rk
+ cat /tmp/tmp.PSrYnBLJmj
+ rm /tmp/tmp.1Njd1oT7Rk /tmp/tmp.PSrYnBLJmj
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml /tmp/tmp.pAywTItlYw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1201
+ local resource=statefulset/some-name-cfg
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml
+ local new_result=/tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5481", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.Vnpq4NVvNp
++ mktemp
+ local LAST_ERR=/tmp/tmp.tdVIPZ8kw3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vnpq4NVvNp
+ cat /tmp/tmp.tdVIPZ8kw3
+ rm /tmp/tmp.Vnpq4NVvNp /tmp/tmp.tdVIPZ8kw3
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml /tmp/tmp.pAywTItlYw/statefulset_some-name-cfg.yml
+ desc 'test 1.21.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.21.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.21.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3KDOJ9Vyqq
++ mktemp
+ local LAST_ERR=/tmp/tmp.NvTCfANTU4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.21.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3KDOJ9Vyqq
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.NvTCfANTU4
+ rm /tmp/tmp.3KDOJ9Vyqq /tmp/tmp.NvTCfANTU4
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0c8cstuRS8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dXBPeSrBIz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0c8cstuRS8
++ cat /tmp/tmp.dXBPeSrBIz
++ rm /tmp/tmp.0c8cstuRS8 /tmp/tmp.dXBPeSrBIz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DLQKIezqvy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aE0khgefpg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DLQKIezqvy
++ cat /tmp/tmp.aE0khgefpg
++ rm /tmp/tmp.DLQKIezqvy /tmp/tmp.aE0khgefpg
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bxz01yuM1V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GX6egMaGOU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bxz01yuM1V
++ cat /tmp/tmp.GX6egMaGOU
++ rm /tmp/tmp.bxz01yuM1V /tmp/tmp.GX6egMaGOU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D4ntDXfrqR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8U7He0AIM8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D4ntDXfrqR
++ cat /tmp/tmp.8U7He0AIM8
++ rm /tmp/tmp.D4ntDXfrqR /tmp/tmp.8U7He0AIM8
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.09ONkke3Sn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pfpJIYauAs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.09ONkke3Sn
++ cat /tmp/tmp.pfpJIYauAs
++ rm /tmp/tmp.09ONkke3Sn /tmp/tmp.pfpJIYauAs
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IiA63deam7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y4ryiup0ko
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IiA63deam7
++ cat /tmp/tmp.y4ryiup0ko
++ rm /tmp/tmp.IiA63deam7 /tmp/tmp.y4ryiup0ko
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10