Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/upgrade-consistency-sharded-tls.log
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ CLUSTER=some-name
+ main
+ create_infra upgrade-consistency-sharded-tls-5186
+ local ns=upgrade-consistency-sharded-tls-5186
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.5zFwMHyuLz
++ mktemp
+ local LAST_ERR=/tmp/tmp.xtZTEHvvTv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5zFwMHyuLz
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.xtZTEHvvTv
+ rm /tmp/tmp.5zFwMHyuLz /tmp/tmp.xtZTEHvvTv
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.BL02hRG0zI
++ mktemp
+ local LAST_ERR=/tmp/tmp.ukGLUT6vGk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BL02hRG0zI
+ cat /tmp/tmp.ukGLUT6vGk
+ rm /tmp/tmp.BL02hRG0zI /tmp/tmp.ukGLUT6vGk
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.samLg5VUzU
++ mktemp
+ local LAST_ERR=/tmp/tmp.5lnIeeyf1O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.samLg5VUzU
+ cat /tmp/tmp.5lnIeeyf1O
+ rm /tmp/tmp.samLg5VUzU /tmp/tmp.5lnIeeyf1O
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-652 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.jP9dBeqSZ8
++ mktemp
+ local LAST_ERR=/tmp/tmp.FQphJacYY7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jP9dBeqSZ8
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.FQphJacYY7
+ rm /tmp/tmp.jP9dBeqSZ8 /tmp/tmp.FQphJacYY7
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.xQs8VPGmWJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.yIL0O9lBeB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xQs8VPGmWJ
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.yIL0O9lBeB
+ rm /tmp/tmp.xQs8VPGmWJ /tmp/tmp.yIL0O9lBeB
+ return 0
+ check_crd_for_deletion PR-1961-5b0cc669
+ local git_tag=PR-1961-5b0cc669
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-5b0cc669/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nlTyMWRFcr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0eQW3t6eFa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nlTyMWRFcr
++ cat /tmp/tmp.0eQW3t6eFa
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nlTyMWRFcr
++ cat /tmp/tmp.0eQW3t6eFa
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.nlTyMWRFcr
++ cat /tmp/tmp.0eQW3t6eFa
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.nlTyMWRFcr
++ cat /tmp/tmp.0eQW3t6eFa
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.nlTyMWRFcr /tmp/tmp.0eQW3t6eFa
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator++ mktemp

-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.s5ZKMUJ35q
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.QIhP8qDzma
++ mktemp
+ local LAST_ERR=/tmp/tmp.kouveHsBAO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.0mzgo1cB7W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s5ZKMUJ35q
+ cat /tmp/tmp.kouveHsBAO
+ rm /tmp/tmp.s5ZKMUJ35q /tmp/tmp.kouveHsBAO
+ return 0
namespace "cert-manager" deleted
namespace "upgrade-consistency-sharded-tls-652" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QIhP8qDzma
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.0mzgo1cB7W
+ rm /tmp/tmp.QIhP8qDzma /tmp/tmp.0mzgo1cB7W
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.TZG0U5yRST
++ mktemp
+ local LAST_ERR=/tmp/tmp.OfSJ5qPyqb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TZG0U5yRST
+ cat /tmp/tmp.OfSJ5qPyqb
+ rm /tmp/tmp.TZG0U5yRST /tmp/tmp.OfSJ5qPyqb
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.EE29ab1kJS
++ mktemp
+ local LAST_ERR=/tmp/tmp.rsnnwz4NRK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EE29ab1kJS
namespace/psmdb-operator created
+ cat /tmp/tmp.rsnnwz4NRK
+ rm /tmp/tmp.EE29ab1kJS /tmp/tmp.rsnnwz4NRK
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AQrV0ZFspu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YTwfNCls4V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AQrV0ZFspu
++ cat /tmp/tmp.YTwfNCls4V
++ rm /tmp/tmp.AQrV0ZFspu /tmp/tmp.YTwfNCls4V
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wq6FjrRRwe
++ mktemp
+ local LAST_ERR=/tmp/tmp.BpKYFfEL7M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wq6FjrRRwe
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10" modified.
+ cat /tmp/tmp.BpKYFfEL7M
+ rm /tmp/tmp.wq6FjrRRwe /tmp/tmp.BpKYFfEL7M
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-5b0cc669'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-5b0cc669
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.V1MQBJH67n
++ mktemp
+ local LAST_ERR=/tmp/tmp.6dt1HiHkzb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V1MQBJH67n
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.6dt1HiHkzb
+ rm /tmp/tmp.V1MQBJH67n /tmp/tmp.6dt1HiHkzb
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vxt5baUUMQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.td1Tz5l6yM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vxt5baUUMQ
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.td1Tz5l6yM
+ rm /tmp/tmp.Vxt5baUUMQ /tmp/tmp.td1Tz5l6yM
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-5b0cc669") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KuS70vmj2R
++ mktemp
+ local LAST_ERR=/tmp/tmp.kdop4zFiyq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KuS70vmj2R
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.kdop4zFiyq
+ rm /tmp/tmp.KuS70vmj2R /tmp/tmp.kdop4zFiyq
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pn0w0iw3zs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RAxcRAJRGc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pn0w0iw3zs
++ cat /tmp/tmp.RAxcRAJRGc
++ rm /tmp/tmp.pn0w0iw3zs /tmp/tmp.RAxcRAJRGc
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-5464c7ccdb-kg98t
+ local pod=percona-server-mongodb-operator-5464c7ccdb-kg98t
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5464c7ccdb-kg98t to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.48GfzLaImI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R1fLSLBXSC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.48GfzLaImI
++ cat /tmp/tmp.R1fLSLBXSC
++ rm /tmp/tmp.48GfzLaImI /tmp/tmp.R1fLSLBXSC
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5464c7ccdb-kg98t
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZH81peI3oj
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kjt8dkS6LN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-5464c7ccdb-kg98t
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZH81peI3oj
+ cat /tmp/tmp.Kjt8dkS6LN
+ rm /tmp/tmp.ZH81peI3oj /tmp/tmp.Kjt8dkS6LN
+ return 0
2025-12-04T21:58:21.753Z	INFO	setup	Manager starting up	{"gitCommit": "5b0cc6694bb3fee55fa4cc155ac44e1b0423e748", "gitBranch": "PR-1961-5b0cc669", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace upgrade-consistency-sharded-tls-5186
+ local namespace=upgrade-consistency-sharded-tls-5186
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-5186'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-sharded-tls-5186
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-sharded-tls-5186 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.E8qfd1MHKH
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.xenANilPBM
+ local LAST_ERR=/tmp/tmp.BnbAp9Yqqy
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.NP2BVKr5M6
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace upgrade-consistency-sharded-tls-5186 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E8qfd1MHKH
+ cat /tmp/tmp.BnbAp9Yqqy
+ rm /tmp/tmp.E8qfd1MHKH /tmp/tmp.BnbAp9Yqqy
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xenANilPBM
+ cat /tmp/tmp.NP2BVKr5M6
+ rm /tmp/tmp.xenANilPBM /tmp/tmp.NP2BVKr5M6
+ return 0
+ kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-5186
++ mktemp
+ local LAST_OUT=/tmp/tmp.LsuzGHfcQV
error: resource(s) were provided, but no name was specified
++ mktemp
+ local LAST_ERR=/tmp/tmp.NTUvDaDHPU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-5186
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LsuzGHfcQV
+ cat /tmp/tmp.NTUvDaDHPU
+ rm /tmp/tmp.LsuzGHfcQV /tmp/tmp.NTUvDaDHPU
+ return 0
+ desc 'create namespace upgrade-consistency-sharded-tls-5186'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-sharded-tls-5186
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-sharded-tls-5186
++ mktemp
+ local LAST_OUT=/tmp/tmp.I7eXSYHqfR
++ mktemp
+ local LAST_ERR=/tmp/tmp.9vk3SH7d0m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace upgrade-consistency-sharded-tls-5186
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I7eXSYHqfR
namespace/upgrade-consistency-sharded-tls-5186 created
+ cat /tmp/tmp.9vk3SH7d0m
+ rm /tmp/tmp.I7eXSYHqfR /tmp/tmp.9vk3SH7d0m
+ return 0
+ set_kube_ctx upgrade-consistency-sharded-tls-5186
+ local namespace=upgrade-consistency-sharded-tls-5186
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yMxDmOFnhu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eeSOY3rQ7W
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yMxDmOFnhu
++ cat /tmp/tmp.eeSOY3rQ7W
++ rm /tmp/tmp.yMxDmOFnhu /tmp/tmp.eeSOY3rQ7W
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10 --namespace=upgrade-consistency-sharded-tls-5186
++ mktemp
+ local LAST_OUT=/tmp/tmp.96JsruFG4a
++ mktemp
+ local LAST_ERR=/tmp/tmp.IVorbwTrK9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10 --namespace=upgrade-consistency-sharded-tls-5186
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.96JsruFG4a
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-5b0cc669-2-cluster10" modified.
+ cat /tmp/tmp.IVorbwTrK9
+ rm /tmp/tmp.96JsruFG4a /tmp/tmp.IVorbwTrK9
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.bDO6OYTgB3
++ mktemp
+ local LAST_ERR=/tmp/tmp.cyoZuHqHM2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bDO6OYTgB3
namespace/cert-manager created
+ cat /tmp/tmp.cyoZuHqHM2
+ rm /tmp/tmp.bDO6OYTgB3 /tmp/tmp.cyoZuHqHM2
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.6tbKKo8cb9
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y5PE3QDXHa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6tbKKo8cb9
namespace/cert-manager labeled
+ cat /tmp/tmp.Y5PE3QDXHa
+ rm /tmp/tmp.6tbKKo8cb9 /tmp/tmp.Y5PE3QDXHa
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.QSGGVOK1w9
++ mktemp
+ local LAST_ERR=/tmp/tmp.DoJ4xWbZys
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.1/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QSGGVOK1w9
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.DoJ4xWbZys
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.QSGGVOK1w9 /tmp/tmp.DoJ4xWbZys
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.WqSb9qZCls
++ mktemp
+ local LAST_ERR=/tmp/tmp.5ZWMTf5owM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WqSb9qZCls
pod/cert-manager-cainjector-5dc9c8b4f7-7nqwh condition met
pod/cert-manager-df4b69479-kz7k8 condition met
pod/cert-manager-webhook-769bbb594d-2npnv condition met
+ cat /tmp/tmp.5ZWMTf5owM
+ rm /tmp/tmp.WqSb9qZCls /tmp/tmp.5ZWMTf5owM
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PBJ3c2xxyh
++ mktemp
+ local LAST_ERR=/tmp/tmp.d0G6Pu3EvJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PBJ3c2xxyh
secret/some-users created
+ cat /tmp/tmp.d0G6Pu3EvJ
+ rm /tmp/tmp.PBJ3c2xxyh /tmp/tmp.d0G6Pu3EvJ
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.qMQTT8bU90
++ mktemp
+ local LAST_ERR=/tmp/tmp.mhXwqqWQED
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qMQTT8bU90
deployment.apps/psmdb-client created
+ cat /tmp/tmp.mhXwqqWQED
+ rm /tmp/tmp.qMQTT8bU90 /tmp/tmp.mhXwqqWQED
+ return 0
+ deploy_cmctl
+ local service_account=cmctl
+ /usr/sbin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/rbac.yaml
+ yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dWRvRy5AYz
++ mktemp
+ local LAST_ERR=/tmp/tmp.uBT2jIpMJ0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dWRvRy5AYz
role.rbac.authorization.k8s.io/cmctl created
serviceaccount/cmctl created
rolebinding.rbac.authorization.k8s.io/service-account-cmctl created
+ cat /tmp/tmp.uBT2jIpMJ0
+ rm /tmp/tmp.dWRvRy5AYz /tmp/tmp.uBT2jIpMJ0
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cmctl.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RZyAjPh6y1
++ mktemp
+ local LAST_ERR=/tmp/tmp.6qIirymRjF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/cmctl.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RZyAjPh6y1
deployment.apps/cmctl created
+ cat /tmp/tmp.6qIirymRjF
+ rm /tmp/tmp.RZyAjPh6y1 /tmp/tmp.6qIirymRjF
+ return 0
+ desc 'create first PSMDB cluster 1.20.1 some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster 1.20.1 some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-5b0cc669"'
+ local LAST_OUT=/tmp/tmp.qrKLj7uo5C
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.11.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.AP2JemK9UZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qrKLj7uo5C
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.AP2JemK9UZ
+ rm /tmp/tmp.qrKLj7uo5C /tmp/tmp.AP2JemK9UZ
+ return 0
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1oWJ0SOFNd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1EM9jGUvvv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1oWJ0SOFNd
++ cat /tmp/tmp.1EM9jGUvvv
++ rm /tmp/tmp.1oWJ0SOFNd /tmp/tmp.1EM9jGUvvv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lm3EWlEL0s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DTzTG5rR3K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lm3EWlEL0s
++ cat /tmp/tmp.DTzTG5rR3K
++ rm /tmp/tmp.Lm3EWlEL0s /tmp/tmp.DTzTG5rR3K
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oISIaYafAc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dk09FUZtYQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oISIaYafAc
++ cat /tmp/tmp.dk09FUZtYQ
++ rm /tmp/tmp.oISIaYafAc /tmp/tmp.dk09FUZtYQ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jdwWxB0epQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1V3gE7DdJp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jdwWxB0epQ
++ cat /tmp/tmp.1V3gE7DdJp
++ rm /tmp/tmp.jdwWxB0epQ /tmp/tmp.1V3gE7DdJp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HSfTxKrHQd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a71k9BoENI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HSfTxKrHQd
++ cat /tmp/tmp.a71k9BoENI
++ rm /tmp/tmp.HSfTxKrHQd /tmp/tmp.a71k9BoENI
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B4DXUz4F07
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2zD5yxBGfZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B4DXUz4F07
++ cat /tmp/tmp.2zD5yxBGfZ
++ rm /tmp/tmp.B4DXUz4F07 /tmp/tmp.2zD5yxBGfZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BkQncKDjJL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zgW1QgHAzi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BkQncKDjJL
++ cat /tmp/tmp.zgW1QgHAzi
++ rm /tmp/tmp.BkQncKDjJL /tmp/tmp.zgW1QgHAzi
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0ucC7RmBmy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O2S48lipZ2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0ucC7RmBmy
++ cat /tmp/tmp.O2S48lipZ2
++ rm /tmp/tmp.0ucC7RmBmy /tmp/tmp.O2S48lipZ2
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.limpN6AWnM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qnGcWrcqyD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.limpN6AWnM
++ cat /tmp/tmp.qnGcWrcqyD
++ rm /tmp/tmp.limpN6AWnM /tmp/tmp.qnGcWrcqyD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A9t9qEzX0N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7J7TTKzqIF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A9t9qEzX0N
++ cat /tmp/tmp.7J7TTKzqIF
++ rm /tmp/tmp.A9t9qEzX0N /tmp/tmp.7J7TTKzqIF
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 1 statefulset some-name-rs0
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUQJV3cSTU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P1WSwQezlN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gUQJV3cSTU
++ cat /tmp/tmp.P1WSwQezlN
++ rm /tmp/tmp.gUQJV3cSTU /tmp/tmp.P1WSwQezlN
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ compare_generation 1 statefulset some-name-cfg
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mDk5wDUzhg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tSPdlHHZRa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mDk5wDUzhg
++ cat /tmp/tmp.tSPdlHHZRa
++ rm /tmp/tmp.mDk5wDUzhg /tmp/tmp.tSPdlHHZRa
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ITQAqFZICL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mvlDGVZVVa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ITQAqFZICL
++ cat /tmp/tmp.mvlDGVZVVa
++ rm /tmp/tmp.ITQAqFZICL /tmp/tmp.mvlDGVZVVa
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UnoEQbMxnv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vrFJMJaTKC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UnoEQbMxnv
++ cat /tmp/tmp.vrFJMJaTKC
++ rm /tmp/tmp.UnoEQbMxnv /tmp/tmp.vrFJMJaTKC
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vB6b2whoF7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GOluVHlgcg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vB6b2whoF7
++ cat /tmp/tmp.GOluVHlgcg
++ rm /tmp/tmp.vB6b2whoF7 /tmp/tmp.GOluVHlgcg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RfjTfUZ90c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OiuKDCnhwP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RfjTfUZ90c
++ cat /tmp/tmp.OiuKDCnhwP
++ rm /tmp/tmp.RfjTfUZ90c /tmp/tmp.OiuKDCnhwP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Jjz7825t2Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WHGk1g0XZx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Jjz7825t2Z
++ cat /tmp/tmp.WHGk1g0XZx
++ rm /tmp/tmp.Jjz7825t2Z /tmp/tmp.WHGk1g0XZx
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VJNbkj8GXE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.74JfJeqUNY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VJNbkj8GXE
++ cat /tmp/tmp.74JfJeqUNY
++ rm /tmp/tmp.VJNbkj8GXE /tmp/tmp.74JfJeqUNY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KzrWpeT3oR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MDfh0IQ0NE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KzrWpeT3oR
++ cat /tmp/tmp.MDfh0IQ0NE
++ rm /tmp/tmp.KzrWpeT3oR /tmp/tmp.MDfh0IQ0NE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pJ3fZtZqpM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ldORKPxkzp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pJ3fZtZqpM
++ cat /tmp/tmp.ldORKPxkzp
++ rm /tmp/tmp.pJ3fZtZqpM /tmp/tmp.ldORKPxkzp
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tB8P3CRuxv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SqNtrH7keQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tB8P3CRuxv
++ cat /tmp/tmp.SqNtrH7keQ
++ rm /tmp/tmp.tB8P3CRuxv /tmp/tmp.SqNtrH7keQ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.be0U7qnagl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uJk7GqVt8c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.be0U7qnagl
++ cat /tmp/tmp.uJk7GqVt8c
++ rm /tmp/tmp.be0U7qnagl /tmp/tmp.uJk7GqVt8c
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5b08YGRCRa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lpNYu7eviq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5b08YGRCRa
++ cat /tmp/tmp.lpNYu7eviq
++ rm /tmp/tmp.5b08YGRCRa /tmp/tmp.lpNYu7eviq
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vdU5tKbxWb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hWuQXjEshA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vdU5tKbxWb
++ cat /tmp/tmp.hWuQXjEshA
++ rm /tmp/tmp.vdU5tKbxWb /tmp/tmp.hWuQXjEshA
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.LlKSvZV9Ml
++ mktemp
+ local LAST_ERR=/tmp/tmp.8RLzDtVilZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LlKSvZV9Ml
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl
+ cat /tmp/tmp.8RLzDtVilZ
+ rm /tmp/tmp.LlKSvZV9Ml /tmp/tmp.8RLzDtVilZ
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NbwANFiTq9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4HO3nCydAd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NbwANFiTq9
++ cat /tmp/tmp.4HO3nCydAd
++ rm /tmp/tmp.NbwANFiTq9 /tmp/tmp.4HO3nCydAd
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Anv4EKfF0U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WVHF7weRn3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Anv4EKfF0U
++ cat /tmp/tmp.WVHF7weRn3
++ rm /tmp/tmp.Anv4EKfF0U /tmp/tmp.WVHF7weRn3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xtQTH7TfIq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BKNW4DXQly
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xtQTH7TfIq
++ cat /tmp/tmp.BKNW4DXQly
++ rm /tmp/tmp.xtQTH7TfIq /tmp/tmp.BKNW4DXQly
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bzsDe78WKe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z9ilXXqjTa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bzsDe78WKe
++ cat /tmp/tmp.z9ilXXqjTa
++ rm /tmp/tmp.bzsDe78WKe /tmp/tmp.z9ilXXqjTa
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5gj9sXXc88
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qOJX93rtpa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5gj9sXXc88
++ cat /tmp/tmp.qOJX93rtpa
++ rm /tmp/tmp.5gj9sXXc88 /tmp/tmp.qOJX93rtpa
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4ScIbFt61n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bIyzuLXB0d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4ScIbFt61n
++ cat /tmp/tmp.bIyzuLXB0d
++ rm /tmp/tmp.4ScIbFt61n /tmp/tmp.bIyzuLXB0d
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PlIvX2ODaF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5SspH5Yh3e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PlIvX2ODaF
++ cat /tmp/tmp.5SspH5Yh3e
++ rm /tmp/tmp.PlIvX2ODaF /tmp/tmp.5SspH5Yh3e
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.syV14ZMuGs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KLkiYP4yBx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.syV14ZMuGs
++ cat /tmp/tmp.KLkiYP4yBx
++ rm /tmp/tmp.syV14ZMuGs /tmp/tmp.KLkiYP4yBx
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c12W1cjTVq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6WcMkaj5sg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c12W1cjTVq
++ cat /tmp/tmp.6WcMkaj5sg
++ rm /tmp/tmp.c12W1cjTVq /tmp/tmp.6WcMkaj5sg
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RoDMHu3ywN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eqUvlK5521
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RoDMHu3ywN
++ cat /tmp/tmp.eqUvlK5521
++ rm /tmp/tmp.RoDMHu3ywN /tmp/tmp.eqUvlK5521
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2vbTV34m9n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0HTf0p7yTk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2vbTV34m9n
++ cat /tmp/tmp.0HTf0p7yTk
++ rm /tmp/tmp.2vbTV34m9n /tmp/tmp.0HTf0p7yTk
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 2 statefulset some-name-rs0
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fppXvdZbfh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6QKe7dqf9w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fppXvdZbfh
++ cat /tmp/tmp.6QKe7dqf9w
++ rm /tmp/tmp.fppXvdZbfh /tmp/tmp.6QKe7dqf9w
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ compare_generation 2 statefulset some-name-cfg
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BsIVPG8Tsj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OmHqlQOmKh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BsIVPG8Tsj
++ cat /tmp/tmp.OmHqlQOmKh
++ rm /tmp/tmp.BsIVPG8Tsj /tmp/tmp.OmHqlQOmKh
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ba0txGi97b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7htdOScKu2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ba0txGi97b
++ cat /tmp/tmp.7htdOScKu2
++ rm /tmp/tmp.ba0txGi97b /tmp/tmp.7htdOScKu2
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IvjkJ9AWNJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wrckUG5Sot
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IvjkJ9AWNJ
++ cat /tmp/tmp.wrckUG5Sot
++ rm /tmp/tmp.IvjkJ9AWNJ /tmp/tmp.wrckUG5Sot
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.UEXIvXfyj0
++ mktemp
+ local LAST_ERR=/tmp/tmp.pdB7Pu3GWk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UEXIvXfyj0
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl-internal
+ cat /tmp/tmp.pdB7Pu3GWk
+ rm /tmp/tmp.UEXIvXfyj0 /tmp/tmp.pdB7Pu3GWk
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GW3Yj8YwKB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OHpIrh0oAw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GW3Yj8YwKB
++ cat /tmp/tmp.OHpIrh0oAw
++ rm /tmp/tmp.GW3Yj8YwKB /tmp/tmp.OHpIrh0oAw
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Ax22LX55O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x7J4JuHlOr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Ax22LX55O
++ cat /tmp/tmp.x7J4JuHlOr
++ rm /tmp/tmp.5Ax22LX55O /tmp/tmp.x7J4JuHlOr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4vrg7PRqfl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fvjEiZNWiE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4vrg7PRqfl
++ cat /tmp/tmp.fvjEiZNWiE
++ rm /tmp/tmp.4vrg7PRqfl /tmp/tmp.fvjEiZNWiE
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9hgkGKj3jL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ow5WoLHTwy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9hgkGKj3jL
++ cat /tmp/tmp.ow5WoLHTwy
++ rm /tmp/tmp.9hgkGKj3jL /tmp/tmp.ow5WoLHTwy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z1guVSmKc5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tg9JMUYeVO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z1guVSmKc5
++ cat /tmp/tmp.Tg9JMUYeVO
++ rm /tmp/tmp.z1guVSmKc5 /tmp/tmp.Tg9JMUYeVO
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6cLNVTX8Kw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q13qfKfUtA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6cLNVTX8Kw
++ cat /tmp/tmp.q13qfKfUtA
++ rm /tmp/tmp.6cLNVTX8Kw /tmp/tmp.q13qfKfUtA
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LTd9nLoG0q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5lOlNYd76X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LTd9nLoG0q
++ cat /tmp/tmp.5lOlNYd76X
++ rm /tmp/tmp.LTd9nLoG0q /tmp/tmp.5lOlNYd76X
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8eFpiIvNbr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ONreBbcLWT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8eFpiIvNbr
++ cat /tmp/tmp.ONreBbcLWT
++ rm /tmp/tmp.8eFpiIvNbr /tmp/tmp.ONreBbcLWT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AVkPxTcmLK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q6yExfGRum
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AVkPxTcmLK
++ cat /tmp/tmp.Q6yExfGRum
++ rm /tmp/tmp.AVkPxTcmLK /tmp/tmp.Q6yExfGRum
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gR9Aj6Fznh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0mA6NxqSkP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gR9Aj6Fznh
++ cat /tmp/tmp.0mA6NxqSkP
++ rm /tmp/tmp.gR9Aj6Fznh /tmp/tmp.0mA6NxqSkP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8OP0yzPoDP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qxDA3ZXzSv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8OP0yzPoDP
++ cat /tmp/tmp.qxDA3ZXzSv
++ rm /tmp/tmp.8OP0yzPoDP /tmp/tmp.qxDA3ZXzSv
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wDOc51eC0L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.81hjTe0M02
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wDOc51eC0L
++ cat /tmp/tmp.81hjTe0M02
++ rm /tmp/tmp.wDOc51eC0L /tmp/tmp.81hjTe0M02
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YXaEWawsQb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zUxvE5hw1T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YXaEWawsQb
++ cat /tmp/tmp.zUxvE5hw1T
++ rm /tmp/tmp.YXaEWawsQb /tmp/tmp.zUxvE5hw1T
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1201
+ local resource=service/some-name-rs0
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XFrEUQ6hUR
++ mktemp
+ local LAST_ERR=/tmp/tmp.VZ0UbytzrK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XFrEUQ6hUR
+ cat /tmp/tmp.VZ0UbytzrK
+ rm /tmp/tmp.XFrEUQ6hUR /tmp/tmp.VZ0UbytzrK
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1201.yml /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2025-12-04T22:19:05+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1201
+ local resource=service/some-name-cfg
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.MikqQGxdH1
++ mktemp
+ local LAST_ERR=/tmp/tmp.IKqIsYDZWV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MikqQGxdH1
+ cat /tmp/tmp.IKqIsYDZWV
+ rm /tmp/tmp.MikqQGxdH1 /tmp/tmp.IKqIsYDZWV
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1201.yml /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2025-12-04T22:19:06+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1201
+ local resource=statefulset/some-name-rs0
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.zHJ1mkfWR9
++ mktemp
+ local LAST_ERR=/tmp/tmp.dFJcbWrIUQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zHJ1mkfWR9
+ cat /tmp/tmp.dFJcbWrIUQ
+ rm /tmp/tmp.zHJ1mkfWR9 /tmp/tmp.dFJcbWrIUQ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1201.yml /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-12-04T22:19:07+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1201
+ local resource=statefulset/some-name-cfg
+ local postfix=-1201
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.i1XV9jynz6
++ mktemp
+ local LAST_ERR=/tmp/tmp.n3FTXXwNoR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i1XV9jynz6
+ cat /tmp/tmp.n3FTXXwNoR
+ rm /tmp/tmp.i1XV9jynz6 /tmp/tmp.n3FTXXwNoR
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1201.yml /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2025-12-04T22:19:08+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'test 1.21.1'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.21.1
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.21.1"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.pAq51LhOU8
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZtT0q0AAax
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.21.1"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pAq51LhOU8
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.ZtT0q0AAax
+ rm /tmp/tmp.pAq51LhOU8 /tmp/tmp.ZtT0q0AAax
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XqhDNSIFW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Z8xNohba9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5XqhDNSIFW
++ cat /tmp/tmp.9Z8xNohba9
++ rm /tmp/tmp.5XqhDNSIFW /tmp/tmp.9Z8xNohba9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iXmPTMfRI4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nutOA7LZ2q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iXmPTMfRI4
++ cat /tmp/tmp.nutOA7LZ2q
++ rm /tmp/tmp.iXmPTMfRI4 /tmp/tmp.nutOA7LZ2q
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MWW8ZyYGC3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HeaJjhelQP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MWW8ZyYGC3
++ cat /tmp/tmp.HeaJjhelQP
++ rm /tmp/tmp.MWW8ZyYGC3 /tmp/tmp.HeaJjhelQP
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...........................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tvYrJoRH1L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4w1kS6vltV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tvYrJoRH1L
++ cat /tmp/tmp.4w1kS6vltV
++ rm /tmp/tmp.tvYrJoRH1L /tmp/tmp.4w1kS6vltV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Th4Sl2fG5d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FkyGYXDv2m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Th4Sl2fG5d
++ cat /tmp/tmp.FkyGYXDv2m
++ rm /tmp/tmp.Th4Sl2fG5d /tmp/tmp.FkyGYXDv2m
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5nIcJ2uS5K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F24wBFYUoe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5nIcJ2uS5K
++ cat /tmp/tmp.F24wBFYUoe
++ rm /tmp/tmp.5nIcJ2uS5K /tmp/tmp.F24wBFYUoe
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LBndTJ1NMM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9OBudiXVwd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LBndTJ1NMM
++ cat /tmp/tmp.9OBudiXVwd
++ rm /tmp/tmp.LBndTJ1NMM /tmp/tmp.9OBudiXVwd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JhZe4fQVq6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Bx50L3olW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JhZe4fQVq6
++ cat /tmp/tmp.7Bx50L3olW
++ rm /tmp/tmp.JhZe4fQVq6 /tmp/tmp.7Bx50L3olW
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0ipvvKUYBc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FWbRFRUbQN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0ipvvKUYBc
++ cat /tmp/tmp.FWbRFRUbQN
++ rm /tmp/tmp.0ipvvKUYBc /tmp/tmp.FWbRFRUbQN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XPTFA9YT9C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PdlQkDx1I1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XPTFA9YT9C
++ cat /tmp/tmp.PdlQkDx1I1
++ rm /tmp/tmp.XPTFA9YT9C /tmp/tmp.PdlQkDx1I1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 4 statefulset some-name-rs0
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O5HXfyxGfw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C0VhIPVjsR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O5HXfyxGfw
++ cat /tmp/tmp.C0VhIPVjsR
++ rm /tmp/tmp.O5HXfyxGfw /tmp/tmp.C0VhIPVjsR
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ compare_generation 4 statefulset some-name-cfg
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fZdE8ffAmo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NY5f3upTZE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fZdE8ffAmo
++ cat /tmp/tmp.NY5f3upTZE
++ rm /tmp/tmp.fZdE8ffAmo /tmp/tmp.NY5f3upTZE
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sPRghVu2Lq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GrMUvLhZ8N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sPRghVu2Lq
++ cat /tmp/tmp.GrMUvLhZ8N
++ rm /tmp/tmp.sPRghVu2Lq /tmp/tmp.GrMUvLhZ8N
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XoQKNhh3LZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gjn2CACNU4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XoQKNhh3LZ
++ cat /tmp/tmp.Gjn2CACNU4
++ rm /tmp/tmp.XoQKNhh3LZ /tmp/tmp.Gjn2CACNU4
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.4RTEppicLh
++ mktemp
+ local LAST_ERR=/tmp/tmp.0aB4dg3P5Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4RTEppicLh
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl
+ cat /tmp/tmp.0aB4dg3P5Q
+ rm /tmp/tmp.4RTEppicLh /tmp/tmp.0aB4dg3P5Q
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IyHxx4EbwD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0t7ZFSD8Nb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IyHxx4EbwD
++ cat /tmp/tmp.0t7ZFSD8Nb
++ rm /tmp/tmp.IyHxx4EbwD /tmp/tmp.0t7ZFSD8Nb
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Y6e3fqiukj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vc82ikvNyY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Y6e3fqiukj
++ cat /tmp/tmp.Vc82ikvNyY
++ rm /tmp/tmp.Y6e3fqiukj /tmp/tmp.Vc82ikvNyY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w83yZqnhhZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Npt3b7nl8P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w83yZqnhhZ
++ cat /tmp/tmp.Npt3b7nl8P
++ rm /tmp/tmp.w83yZqnhhZ /tmp/tmp.Npt3b7nl8P
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tnx7j4yrAf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WVGQnPQTol
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tnx7j4yrAf
++ cat /tmp/tmp.WVGQnPQTol
++ rm /tmp/tmp.tnx7j4yrAf /tmp/tmp.WVGQnPQTol
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kigYlSZonE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sPqDfbAjPw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kigYlSZonE
++ cat /tmp/tmp.sPqDfbAjPw
++ rm /tmp/tmp.kigYlSZonE /tmp/tmp.sPqDfbAjPw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QQTQ4BXNfo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fk1KvRRofx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QQTQ4BXNfo
++ cat /tmp/tmp.Fk1KvRRofx
++ rm /tmp/tmp.QQTQ4BXNfo /tmp/tmp.Fk1KvRRofx
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wh5GsjDYmE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dmyjWnlHkI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wh5GsjDYmE
++ cat /tmp/tmp.dmyjWnlHkI
++ rm /tmp/tmp.wh5GsjDYmE /tmp/tmp.dmyjWnlHkI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MNlzQ22CwC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jMxiXwDoUw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MNlzQ22CwC
++ cat /tmp/tmp.jMxiXwDoUw
++ rm /tmp/tmp.MNlzQ22CwC /tmp/tmp.jMxiXwDoUw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ccLH7o3oh8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gBnHvl6g9l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ccLH7o3oh8
++ cat /tmp/tmp.gBnHvl6g9l
++ rm /tmp/tmp.ccLH7o3oh8 /tmp/tmp.gBnHvl6g9l
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xWgYEdrLPi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3Gcvcz3oTT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xWgYEdrLPi
++ cat /tmp/tmp.3Gcvcz3oTT
++ rm /tmp/tmp.xWgYEdrLPi /tmp/tmp.3Gcvcz3oTT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XirZ4PbA7t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i34O8OJLPm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XirZ4PbA7t
++ cat /tmp/tmp.i34O8OJLPm
++ rm /tmp/tmp.XirZ4PbA7t /tmp/tmp.i34O8OJLPm
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 5 statefulset some-name-rs0
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tnXVdoBIMr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dMnhHnrdsm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tnXVdoBIMr
++ cat /tmp/tmp.dMnhHnrdsm
++ rm /tmp/tmp.tnXVdoBIMr /tmp/tmp.dMnhHnrdsm
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ compare_generation 5 statefulset some-name-cfg
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2YwyMPk1TD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rGpVBx4aGA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2YwyMPk1TD
++ cat /tmp/tmp.rGpVBx4aGA
++ rm /tmp/tmp.2YwyMPk1TD /tmp/tmp.rGpVBx4aGA
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D8MGXXGEim
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f6oTmOWn3q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D8MGXXGEim
++ cat /tmp/tmp.f6oTmOWn3q
++ rm /tmp/tmp.D8MGXXGEim /tmp/tmp.f6oTmOWn3q
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lqq7bI5mkZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2vB4SAZyHa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lqq7bI5mkZ
++ cat /tmp/tmp.2vB4SAZyHa
++ rm /tmp/tmp.lqq7bI5mkZ /tmp/tmp.2vB4SAZyHa
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.t0GZnY1dah
++ mktemp
+ local LAST_ERR=/tmp/tmp.5iOjnP2wHB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t0GZnY1dah
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl-internal
+ cat /tmp/tmp.5iOjnP2wHB
+ rm /tmp/tmp.t0GZnY1dah /tmp/tmp.5iOjnP2wHB
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xww5wLp612
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uQIPvDpF90
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xww5wLp612
++ cat /tmp/tmp.uQIPvDpF90
++ rm /tmp/tmp.Xww5wLp612 /tmp/tmp.uQIPvDpF90
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RcQg52VidM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.05YT0FQsQJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RcQg52VidM
++ cat /tmp/tmp.05YT0FQsQJ
++ rm /tmp/tmp.RcQg52VidM /tmp/tmp.05YT0FQsQJ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MwwysOImIK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dsf55CZ4KK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MwwysOImIK
++ cat /tmp/tmp.dsf55CZ4KK
++ rm /tmp/tmp.MwwysOImIK /tmp/tmp.dsf55CZ4KK
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fHNfVQVO2M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8DKcdeJ9I5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fHNfVQVO2M
++ cat /tmp/tmp.8DKcdeJ9I5
++ rm /tmp/tmp.fHNfVQVO2M /tmp/tmp.8DKcdeJ9I5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OpqxGcWzQl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XBS4BnrOVy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OpqxGcWzQl
++ cat /tmp/tmp.XBS4BnrOVy
++ rm /tmp/tmp.OpqxGcWzQl /tmp/tmp.XBS4BnrOVy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jPm3Fld0dt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HpRj2NTEKU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jPm3Fld0dt
++ cat /tmp/tmp.HpRj2NTEKU
++ rm /tmp/tmp.jPm3Fld0dt /tmp/tmp.HpRj2NTEKU
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZMkwPo3QE1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5HyZahxq2a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZMkwPo3QE1
++ cat /tmp/tmp.5HyZahxq2a
++ rm /tmp/tmp.ZMkwPo3QE1 /tmp/tmp.5HyZahxq2a
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lSZsEZoKx6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O37eCNb2Aq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lSZsEZoKx6
++ cat /tmp/tmp.O37eCNb2Aq
++ rm /tmp/tmp.lSZsEZoKx6 /tmp/tmp.O37eCNb2Aq
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n4jZWbT2nt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x4lXjHF7eQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n4jZWbT2nt
++ cat /tmp/tmp.x4lXjHF7eQ
++ rm /tmp/tmp.n4jZWbT2nt /tmp/tmp.x4lXjHF7eQ
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CjXVnWB7dH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HP36QeyY03
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CjXVnWB7dH
++ cat /tmp/tmp.HP36QeyY03
++ rm /tmp/tmp.CjXVnWB7dH /tmp/tmp.HP36QeyY03
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CeHoKldWAq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v8k8RRFAfh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CeHoKldWAq
++ cat /tmp/tmp.v8k8RRFAfh
++ rm /tmp/tmp.CeHoKldWAq /tmp/tmp.v8k8RRFAfh
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jKzIP07uZm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2v6yWsSLRP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jKzIP07uZm
++ cat /tmp/tmp.2v6yWsSLRP
++ rm /tmp/tmp.jKzIP07uZm /tmp/tmp.2v6yWsSLRP
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LqNJKflGVu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0CfQIAZKZG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LqNJKflGVu
++ cat /tmp/tmp.0CfQIAZKZG
++ rm /tmp/tmp.LqNJKflGVu /tmp/tmp.0CfQIAZKZG
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1211
+ local resource=service/some-name-rs0
+ local postfix=-1211
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1211.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1211-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.L7pWW3hpy0
++ mktemp
+ local LAST_ERR=/tmp/tmp.afbmjommyM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.L7pWW3hpy0
+ cat /tmp/tmp.afbmjommyM
+ rm /tmp/tmp.L7pWW3hpy0 /tmp/tmp.afbmjommyM
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1211.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1211.yml /tmp/tmp.2iq6KmKIdn/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2025-12-04T22:36:49+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1211
+ local resource=service/some-name-cfg
+ local postfix=-1211
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1211.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1211-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.uHBKrIMyGK
++ mktemp
+ local LAST_ERR=/tmp/tmp.G4ZAzJgrVF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uHBKrIMyGK
+ cat /tmp/tmp.G4ZAzJgrVF
+ rm /tmp/tmp.uHBKrIMyGK /tmp/tmp.G4ZAzJgrVF
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1211.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1211.yml /tmp/tmp.2iq6KmKIdn/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2025-12-04T22:36:49+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1211
+ local resource=statefulset/some-name-rs0
+ local postfix=-1211
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1211.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1211-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dov7PzMkJ9
++ mktemp
+ local LAST_ERR=/tmp/tmp.lSCljokd6h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dov7PzMkJ9
+ cat /tmp/tmp.lSCljokd6h
+ rm /tmp/tmp.dov7PzMkJ9 /tmp/tmp.lSCljokd6h
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1211.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1211.yml /tmp/tmp.2iq6KmKIdn/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-12-04T22:36:50+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1211
+ local resource=statefulset/some-name-cfg
+ local postfix=-1211
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1211.yml
+ local new_result=/tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1211-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-5186", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5m5j6wwMP3
++ mktemp
+ local LAST_ERR=/tmp/tmp.ryahd8R2po
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5m5j6wwMP3
+ cat /tmp/tmp.ryahd8R2po
+ rm /tmp/tmp.5m5j6wwMP3 /tmp/tmp.ryahd8R2po
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1211.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1211.yml /tmp/tmp.2iq6KmKIdn/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2025-12-04T22:36:51+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'test 1.22.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.22.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.22.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.dsQVTZIQqM
++ mktemp
+ local LAST_ERR=/tmp/tmp.dNNZ8Jj3cr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.22.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dsQVTZIQqM
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.dNNZ8Jj3cr
+ rm /tmp/tmp.dsQVTZIQqM /tmp/tmp.dNNZ8Jj3cr
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ggNzXMGBdV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o1oQKY5Ngu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ggNzXMGBdV
++ cat /tmp/tmp.o1oQKY5Ngu
++ rm /tmp/tmp.ggNzXMGBdV /tmp/tmp.o1oQKY5Ngu
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tPzp4Uy5wd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cm6boAJhcM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tPzp4Uy5wd
++ cat /tmp/tmp.cm6boAJhcM
++ rm /tmp/tmp.tPzp4Uy5wd /tmp/tmp.cm6boAJhcM
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bj8weTUeGC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9s5nR7Aacd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bj8weTUeGC
++ cat /tmp/tmp.9s5nR7Aacd
++ rm /tmp/tmp.bj8weTUeGC /tmp/tmp.9s5nR7Aacd
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUQGlnIZIX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZUSM8wSO6h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gUQGlnIZIX
++ cat /tmp/tmp.ZUSM8wSO6h
++ rm /tmp/tmp.gUQGlnIZIX /tmp/tmp.ZUSM8wSO6h
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KGSGQUXSZb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bBwdExY7TF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KGSGQUXSZb
++ cat /tmp/tmp.bBwdExY7TF
++ rm /tmp/tmp.KGSGQUXSZb /tmp/tmp.bBwdExY7TF
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JoXzsCOEUe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jw90R9Ln4m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JoXzsCOEUe
++ cat /tmp/tmp.Jw90R9Ln4m
++ rm /tmp/tmp.JoXzsCOEUe /tmp/tmp.Jw90R9Ln4m
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5XUOV81ChB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.duJSMmZLPX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5XUOV81ChB
++ cat /tmp/tmp.duJSMmZLPX
++ rm /tmp/tmp.5XUOV81ChB /tmp/tmp.duJSMmZLPX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wmZS46pji2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NCZfYOQ85F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wmZS46pji2
++ cat /tmp/tmp.NCZfYOQ85F
++ rm /tmp/tmp.wmZS46pji2 /tmp/tmp.NCZfYOQ85F
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vCZySnee96
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0jZAHImbFb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vCZySnee96
++ cat /tmp/tmp.0jZAHImbFb
++ rm /tmp/tmp.vCZySnee96 /tmp/tmp.0jZAHImbFb
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MZwLE6zItT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ttc31O9ujG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MZwLE6zItT
++ cat /tmp/tmp.Ttc31O9ujG
++ rm /tmp/tmp.MZwLE6zItT /tmp/tmp.Ttc31O9ujG
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.smNPRHqij6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q7pYNF3U7v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.smNPRHqij6
++ cat /tmp/tmp.Q7pYNF3U7v
++ rm /tmp/tmp.smNPRHqij6 /tmp/tmp.Q7pYNF3U7v
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G7oDa1uaKB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MgWijpFxqq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G7oDa1uaKB
++ cat /tmp/tmp.MgWijpFxqq
++ rm /tmp/tmp.G7oDa1uaKB /tmp/tmp.MgWijpFxqq
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tIfgngP4YV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.htp5VbinDG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tIfgngP4YV
++ cat /tmp/tmp.htp5VbinDG
++ rm /tmp/tmp.tIfgngP4YV /tmp/tmp.htp5VbinDG
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.201jen31iU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z01GT35CxI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.201jen31iU
++ cat /tmp/tmp.Z01GT35CxI
++ rm /tmp/tmp.201jen31iU /tmp/tmp.Z01GT35CxI
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.m0z9HIyJFQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.yQfyzofMyk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m0z9HIyJFQ
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl
+ cat /tmp/tmp.yQfyzofMyk
+ rm /tmp/tmp.m0z9HIyJFQ /tmp/tmp.yQfyzofMyk
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YCy7jmjzqe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iLMPKkDShy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YCy7jmjzqe
++ cat /tmp/tmp.iLMPKkDShy
++ rm /tmp/tmp.YCy7jmjzqe /tmp/tmp.iLMPKkDShy
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LFqFS4qxb6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6ESMPWmwE8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LFqFS4qxb6
++ cat /tmp/tmp.6ESMPWmwE8
++ rm /tmp/tmp.LFqFS4qxb6 /tmp/tmp.6ESMPWmwE8
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.05V6OBHjNe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wgF63VKKCX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.05V6OBHjNe
++ cat /tmp/tmp.wgF63VKKCX
++ rm /tmp/tmp.05V6OBHjNe /tmp/tmp.wgF63VKKCX
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fJYFdWCtpt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Clpn6pVt7i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fJYFdWCtpt
++ cat /tmp/tmp.Clpn6pVt7i
++ rm /tmp/tmp.fJYFdWCtpt /tmp/tmp.Clpn6pVt7i
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness........................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KItgLBRual
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tjrYw264Q3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KItgLBRual
++ cat /tmp/tmp.tjrYw264Q3
++ rm /tmp/tmp.KItgLBRual /tmp/tmp.tjrYw264Q3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FZrYnHR4Yp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4IvljabXPy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FZrYnHR4Yp
++ cat /tmp/tmp.4IvljabXPy
++ rm /tmp/tmp.FZrYnHR4Yp /tmp/tmp.4IvljabXPy
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ev2ARMbEOJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uUIOtK6CHI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ev2ARMbEOJ
++ cat /tmp/tmp.uUIOtK6CHI
++ rm /tmp/tmp.Ev2ARMbEOJ /tmp/tmp.uUIOtK6CHI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d5HVcapWlf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q6VMbjVgas
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d5HVcapWlf
++ cat /tmp/tmp.Q6VMbjVgas
++ rm /tmp/tmp.d5HVcapWlf /tmp/tmp.Q6VMbjVgas
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w7aArPU4MK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m50dztmJpg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w7aArPU4MK
++ cat /tmp/tmp.m50dztmJpg
++ rm /tmp/tmp.w7aArPU4MK /tmp/tmp.m50dztmJpg
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WnY3EFR9W7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mzVIQAqXPS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WnY3EFR9W7
++ cat /tmp/tmp.mzVIQAqXPS
++ rm /tmp/tmp.WnY3EFR9W7 /tmp/tmp.mzVIQAqXPS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Sa3sc3iu5s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J7ncBFq22F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Sa3sc3iu5s
++ cat /tmp/tmp.J7ncBFq22F
++ rm /tmp/tmp.Sa3sc3iu5s /tmp/tmp.J7ncBFq22F
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ compare_generation 7 statefulset some-name-rs0
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z7bKnaPabG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kj0LQZEURx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z7bKnaPabG
++ cat /tmp/tmp.kj0LQZEURx
++ rm /tmp/tmp.Z7bKnaPabG /tmp/tmp.kj0LQZEURx
++ return 0
+ current_generation=7
+ [[ 7 != \7 ]]
+ compare_generation 7 statefulset some-name-cfg
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oGOymZG18P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tQQTeswu9I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oGOymZG18P
++ cat /tmp/tmp.tQQTeswu9I
++ rm /tmp/tmp.oGOymZG18P /tmp/tmp.tQQTeswu9I
++ return 0
+ current_generation=7
+ [[ 7 != \7 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ve07m6g6es
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tuOlE5dRG9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ve07m6g6es
++ cat /tmp/tmp.tuOlE5dRG9
++ rm /tmp/tmp.Ve07m6g6es /tmp/tmp.tuOlE5dRG9
++ return 0
+ pod_name=cmctl-7c7f6bf77d-xs6p9
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aujGvbLaXW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ffd9DqCHOS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aujGvbLaXW
++ cat /tmp/tmp.ffd9DqCHOS
++ rm /tmp/tmp.aujGvbLaXW /tmp/tmp.ffd9DqCHOS
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.aDZos4thE3
++ mktemp
+ local LAST_ERR=/tmp/tmp.jY4kMcBspS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-7c7f6bf77d-xs6p9 -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aDZos4thE3
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-5186/some-name-ssl-internal
+ cat /tmp/tmp.jY4kMcBspS
+ rm /tmp/tmp.aDZos4thE3 /tmp/tmp.jY4kMcBspS
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kZbQv6I8DM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fbsuZoOqnM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kZbQv6I8DM
++ cat /tmp/tmp.fbsuZoOqnM
++ rm /tmp/tmp.kZbQv6I8DM /tmp/tmp.fbsuZoOqnM
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9sNwkjOApL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xwIItt72bZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9sNwkjOApL
++ cat /tmp/tmp.xwIItt72bZ
++ rm /tmp/tmp.9sNwkjOApL /tmp/tmp.xwIItt72bZ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nXq9N7aWSM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fpd0BroSFA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nXq9N7aWSM
++ cat /tmp/tmp.fpd0BroSFA
++ rm /tmp/tmp.nXq9N7aWSM /tmp/tmp.fpd0BroSFA
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WEznM9AXBd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GJPx6jV7tl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WEznM9AXBd
++ cat /tmp/tmp.GJPx6jV7tl
++ rm /tmp/tmp.WEznM9AXBd /tmp/tmp.GJPx6jV7tl
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...........................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zf8zzW8TA6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZmmV6luTNy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zf8zzW8TA6
++ cat /tmp/tmp.ZmmV6luTNy
++ rm /tmp/tmp.zf8zzW8TA6 /tmp/tmp.ZmmV6luTNy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kyzYnev4mr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jFZWCTUplf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kyzYnev4mr
++ cat /tmp/tmp.jFZWCTUplf
++ rm /tmp/tmp.kyzYnev4mr /tmp/tmp.jFZWCTUplf
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T31KgOE7v9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7bQHzMOjTI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T31KgOE7v9
++ cat /tmp/tmp.7bQHzMOjTI
++ rm /tmp/tmp.T31KgOE7v9 /tmp/tmp.7bQHzMOjTI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10