Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/upgrade-consistency-sharded-tls.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ CLUSTER=some-name
+ main
+ create_infra upgrade-consistency-sharded-tls-22124
+ local ns=upgrade-consistency-sharded-tls-22124
+ echo upgrade-consistency-sharded-tls-22124
+ [[ 0 == 1 ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace

++ mktemp

-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.LfZ05WtfFk
++ mktemp
+ local LAST_OUT=/tmp/tmp.RJEORFxaiE
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wv6RGnFtxd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.aarNbrsdh1
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
++ seq 0 2
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LfZ05WtfFk
+ cat /tmp/tmp.Wv6RGnFtxd
+ rm /tmp/tmp.LfZ05WtfFk /tmp/tmp.Wv6RGnFtxd
+ return 0
namespace "self-healing-chaos-31919" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RJEORFxaiE
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.aarNbrsdh1
+ rm /tmp/tmp.RJEORFxaiE /tmp/tmp.aarNbrsdh1
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.icNmLSOGsA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Sub6kEXw2Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.icNmLSOGsA
+ cat /tmp/tmp.Sub6kEXw2Z
+ rm /tmp/tmp.icNmLSOGsA /tmp/tmp.Sub6kEXw2Z
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YxMSPEX1QD
++ mktemp
+ local LAST_ERR=/tmp/tmp.Gj4Mo05c9H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YxMSPEX1QD
namespace/psmdb-operator created
+ cat /tmp/tmp.Gj4Mo05c9H
+ rm /tmp/tmp.YxMSPEX1QD /tmp/tmp.Gj4Mo05c9H
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IAXZoHDYkk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A1qtKSLuKj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IAXZoHDYkk
++ cat /tmp/tmp.A1qtKSLuKj
++ rm /tmp/tmp.IAXZoHDYkk /tmp/tmp.A1qtKSLuKj
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jgdyHJjxBo
++ mktemp
+ local LAST_ERR=/tmp/tmp.vQw68ffSPS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jgdyHJjxBo
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7" modified.
+ cat /tmp/tmp.vQw68ffSPS
+ rm /tmp/tmp.jgdyHJjxBo /tmp/tmp.vQw68ffSPS
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.pUEfljGCsK
++ mktemp
+ local LAST_ERR=/tmp/tmp.X5Vnf5qdU4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pUEfljGCsK
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.X5Vnf5qdU4
+ rm /tmp/tmp.pUEfljGCsK /tmp/tmp.X5Vnf5qdU4
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wyTLTRv3Sd
++ mktemp
+ local LAST_ERR=/tmp/tmp.QgmuuNWwdc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wyTLTRv3Sd
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged
+ cat /tmp/tmp.QgmuuNWwdc
+ rm /tmp/tmp.wyTLTRv3Sd /tmp/tmp.QgmuuNWwdc
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8C9FzcfgpX
++ mktemp
+ local LAST_ERR=/tmp/tmp.suBKmuE5dq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8C9FzcfgpX
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.suBKmuE5dq
+ rm /tmp/tmp.8C9FzcfgpX /tmp/tmp.suBKmuE5dq
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2tQ8iVpCtI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cvbNc7rodU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2tQ8iVpCtI
++ cat /tmp/tmp.cvbNc7rodU
++ rm /tmp/tmp.2tQ8iVpCtI /tmp/tmp.cvbNc7rodU
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-znw4m
+ local pod=percona-server-mongodb-operator-859f4bc6d7-znw4m
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-859f4bc6d7-znw4m to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TZkpNx9Kz8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bddrSpO7oe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TZkpNx9Kz8
++ cat /tmp/tmp.bddrSpO7oe
++ rm /tmp/tmp.TZkpNx9Kz8 /tmp/tmp.bddrSpO7oe
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-znw4m
++ mktemp
+ local LAST_OUT=/tmp/tmp.RVadngBPCn
++ mktemp
+ local LAST_ERR=/tmp/tmp.pSpem5PlHe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-znw4m
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RVadngBPCn
+ cat /tmp/tmp.pSpem5PlHe
+ rm /tmp/tmp.RVadngBPCn /tmp/tmp.pSpem5PlHe
+ return 0
2026-06-09T10:59:42.033Z	INFO	setup	Manager starting up	{"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace upgrade-consistency-sharded-tls-22124
+ local namespace=upgrade-consistency-sharded-tls-22124
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-22124'
+ set +o xtrace


+ awk '{print$1}'
-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-sharded-tls-22124
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-sharded-tls-22124 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.LmQ35vDbUE
+ local LAST_OUT=/tmp/tmp.0fzzqJyanE
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qx7THM03LZ
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.yXtJSXfwsF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace upgrade-consistency-sharded-tls-22124 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0fzzqJyanE
+ cat /tmp/tmp.Qx7THM03LZ
+ rm /tmp/tmp.0fzzqJyanE /tmp/tmp.Qx7THM03LZ
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LmQ35vDbUE
+ cat /tmp/tmp.yXtJSXfwsF
+ rm /tmp/tmp.LmQ35vDbUE /tmp/tmp.yXtJSXfwsF
+ return 0
+ kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-22124
++ mktemp
+ local LAST_OUT=/tmp/tmp.Sm7mIXDnFk
++ mktemp
+ local LAST_ERR=/tmp/tmp.icePQRbYqG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-22124
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Sm7mIXDnFk
+ cat /tmp/tmp.icePQRbYqG
+ rm /tmp/tmp.Sm7mIXDnFk /tmp/tmp.icePQRbYqG
+ return 0
+ desc 'create namespace upgrade-consistency-sharded-tls-22124'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-sharded-tls-22124
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-sharded-tls-22124
++ mktemp
+ local LAST_OUT=/tmp/tmp.kO9g5nGuHQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.GNFtkFZdk1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace upgrade-consistency-sharded-tls-22124
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kO9g5nGuHQ
namespace/upgrade-consistency-sharded-tls-22124 created
+ cat /tmp/tmp.GNFtkFZdk1
+ rm /tmp/tmp.kO9g5nGuHQ /tmp/tmp.GNFtkFZdk1
+ return 0
+ set_kube_ctx upgrade-consistency-sharded-tls-22124
+ local namespace=upgrade-consistency-sharded-tls-22124
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n9zq2bEJ4Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FySjr78pDn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n9zq2bEJ4Q
++ cat /tmp/tmp.FySjr78pDn
++ rm /tmp/tmp.n9zq2bEJ4Q /tmp/tmp.FySjr78pDn
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7 --namespace=upgrade-consistency-sharded-tls-22124
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z7kj3CXn2X
++ mktemp
+ local LAST_ERR=/tmp/tmp.h8agVJCKUU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7 --namespace=upgrade-consistency-sharded-tls-22124
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z7kj3CXn2X
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster7" modified.
+ cat /tmp/tmp.h8agVJCKUU
+ rm /tmp/tmp.Z7kj3CXn2X /tmp/tmp.h8agVJCKUU
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.7Ma4d18dU0
++ mktemp
+ local LAST_ERR=/tmp/tmp.pfGTPXoaOi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7Ma4d18dU0
namespace/cert-manager created
+ cat /tmp/tmp.pfGTPXoaOi
+ rm /tmp/tmp.7Ma4d18dU0 /tmp/tmp.pfGTPXoaOi
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.oNevW1cc5F
++ mktemp
+ local LAST_ERR=/tmp/tmp.jN2CbuaZWr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oNevW1cc5F
namespace/cert-manager labeled
+ cat /tmp/tmp.jN2CbuaZWr
+ rm /tmp/tmp.oNevW1cc5F /tmp/tmp.jN2CbuaZWr
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.MdCrXZmaZy
++ mktemp
+ local LAST_ERR=/tmp/tmp.5bNov0WHaE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MdCrXZmaZy
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io created
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io created
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view created
clusterrole.rbac.authorization.k8s.io/cert-manager-view created
clusterrole.rbac.authorization.k8s.io/cert-manager-edit created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests created
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews created
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created
role.rbac.authorization.k8s.io/cert-manager:leaderelection created
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection created
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection created
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook created
+ cat /tmp/tmp.5bNov0WHaE
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.MdCrXZmaZy /tmp/tmp.5bNov0WHaE
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lnq6uOCzZF
++ mktemp
+ local LAST_ERR=/tmp/tmp.JivFeglIDv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lnq6uOCzZF
pod/cert-manager-559d798845-8nxwn condition met
pod/cert-manager-cainjector-64958d9c7c-ltc4c condition met
pod/cert-manager-webhook-7fb6f99b56-xhvkd condition met
+ cat /tmp/tmp.JivFeglIDv
E0609 11:00:42.863772 1251963 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/api/v1/namespaces/cert-manager/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dcert-manager-cainjector-64958d9c7c-ltc4c&resourceVersion=1781002842478623021&timeoutSeconds=487&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.Lnq6uOCzZF /tmp/tmp.JivFeglIDv
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.QGklgycBC9
++ mktemp
+ local LAST_ERR=/tmp/tmp.ASIWLrY9Bn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QGklgycBC9
secret/some-users created
+ cat /tmp/tmp.ASIWLrY9Bn
+ rm /tmp/tmp.QGklgycBC9 /tmp/tmp.ASIWLrY9Bn
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.6o20Y9ouvr
++ mktemp
+ local LAST_ERR=/tmp/tmp.7DOL0xx4Ep
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6o20Y9ouvr
deployment.apps/psmdb-client created
+ cat /tmp/tmp.7DOL0xx4Ep
+ rm /tmp/tmp.6o20Y9ouvr /tmp/tmp.7DOL0xx4Ep
+ return 0
+ deploy_cmctl
+ local service_account=cmctl
+ /usr/sbin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/rbac.yaml
+ yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2HzrzFxmCo
++ mktemp
+ local LAST_ERR=/tmp/tmp.EBY3I95JSY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2HzrzFxmCo
role.rbac.authorization.k8s.io/cmctl created
serviceaccount/cmctl created
rolebinding.rbac.authorization.k8s.io/service-account-cmctl created
+ cat /tmp/tmp.EBY3I95JSY
+ rm /tmp/tmp.2HzrzFxmCo /tmp/tmp.EBY3I95JSY
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cmctl.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.wopEoAw6Vz
++ mktemp
+ local LAST_ERR=/tmp/tmp.d6RSagVzay
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cmctl.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wopEoAw6Vz
deployment.apps/cmctl created
+ cat /tmp/tmp.d6RSagVzay
+ rm /tmp/tmp.wopEoAw6Vz /tmp/tmp.d6RSagVzay
+ return 0
+ desc 'create first PSMDB cluster 1.21.2 some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster 1.21.2 some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.J4u9uk5xgI
+ /usr/sbin/sed -e s/NAME_SPACE/upgrade-consistency-sharded-tls-22124/g
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.GJyESkrnOa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J4u9uk5xgI
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.GJyESkrnOa
+ rm /tmp/tmp.J4u9uk5xgI /tmp/tmp.GJyESkrnOa
+ return 0
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pEI1OO0p5q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sA6YVwyNi4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pEI1OO0p5q
++ cat /tmp/tmp.sA6YVwyNi4
++ rm /tmp/tmp.pEI1OO0p5q /tmp/tmp.sA6YVwyNi4
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.liyilGn1Uv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zBiBtgpf7D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.liyilGn1Uv
++ cat /tmp/tmp.zBiBtgpf7D
++ rm /tmp/tmp.liyilGn1Uv /tmp/tmp.zBiBtgpf7D
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vDOg2nYXuT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M6vAIdwWn4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vDOg2nYXuT
++ cat /tmp/tmp.M6vAIdwWn4
++ rm /tmp/tmp.vDOg2nYXuT /tmp/tmp.M6vAIdwWn4
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RxUCVDtfE3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tbz17A8dsV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RxUCVDtfE3
++ cat /tmp/tmp.Tbz17A8dsV
++ rm /tmp/tmp.RxUCVDtfE3 /tmp/tmp.Tbz17A8dsV
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sbGRoQn7iY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b1Uc7VvM3m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sbGRoQn7iY
++ cat /tmp/tmp.b1Uc7VvM3m
++ rm /tmp/tmp.sbGRoQn7iY /tmp/tmp.b1Uc7VvM3m
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I0NFAcqJzc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MWLVWnRHPS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I0NFAcqJzc
++ cat /tmp/tmp.MWLVWnRHPS
++ rm /tmp/tmp.I0NFAcqJzc /tmp/tmp.MWLVWnRHPS
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iY8zGjAs59
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qi2c32Sk4X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iY8zGjAs59
++ cat /tmp/tmp.Qi2c32Sk4X
++ rm /tmp/tmp.iY8zGjAs59 /tmp/tmp.Qi2c32Sk4X
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5rLe9GvgwO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O3z5pjFp3R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5rLe9GvgwO
++ cat /tmp/tmp.O3z5pjFp3R
++ rm /tmp/tmp.5rLe9GvgwO /tmp/tmp.O3z5pjFp3R
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8tri5JBTZK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b9uLclJhUk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8tri5JBTZK
++ cat /tmp/tmp.b9uLclJhUk
++ rm /tmp/tmp.8tri5JBTZK /tmp/tmp.b9uLclJhUk
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V4sO1Ws3rm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5oj1VAE4fz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V4sO1Ws3rm
++ cat /tmp/tmp.5oj1VAE4fz
++ rm /tmp/tmp.V4sO1Ws3rm /tmp/tmp.5oj1VAE4fz
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 1 statefulset some-name-rs0
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9WWlNfr6tK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dQdKip0RGu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9WWlNfr6tK
++ cat /tmp/tmp.dQdKip0RGu
++ rm /tmp/tmp.9WWlNfr6tK /tmp/tmp.dQdKip0RGu
++ return 0
+ current_generation=1
+ [[ 1 != 1 ]]
+ compare_generation 1 statefulset some-name-cfg
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CvbBtgJFOw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fbxIbILqcA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CvbBtgJFOw
++ cat /tmp/tmp.fbxIbILqcA
++ rm /tmp/tmp.CvbBtgJFOw /tmp/tmp.fbxIbILqcA
++ return 0
+ current_generation=1
+ [[ 1 != 1 ]]
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rJ21IEmXIy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.chQfiKebf2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rJ21IEmXIy
++ cat /tmp/tmp.chQfiKebf2
++ rm /tmp/tmp.rJ21IEmXIy /tmp/tmp.chQfiKebf2
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8fZzUYBUmb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5VMv2BGiCx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8fZzUYBUmb
++ cat /tmp/tmp.5VMv2BGiCx
++ rm /tmp/tmp.8fZzUYBUmb /tmp/tmp.5VMv2BGiCx
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0hsxgNx3TE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.npaViCdqpj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0hsxgNx3TE
++ cat /tmp/tmp.npaViCdqpj
++ rm /tmp/tmp.0hsxgNx3TE /tmp/tmp.npaViCdqpj
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NG51u9tIBz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QVDLGzWxMd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NG51u9tIBz
++ cat /tmp/tmp.QVDLGzWxMd
++ rm /tmp/tmp.NG51u9tIBz /tmp/tmp.QVDLGzWxMd
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZYX9BA6Ag4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gLUe1jEmby
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZYX9BA6Ag4
++ cat /tmp/tmp.gLUe1jEmby
++ rm /tmp/tmp.ZYX9BA6Ag4 /tmp/tmp.gLUe1jEmby
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VYTNuG3XnX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O0R6YLklls
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VYTNuG3XnX
++ cat /tmp/tmp.O0R6YLklls
++ rm /tmp/tmp.VYTNuG3XnX /tmp/tmp.O0R6YLklls
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1VA0ioyCjs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lH4QLau9y0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1VA0ioyCjs
++ cat /tmp/tmp.lH4QLau9y0
++ rm /tmp/tmp.1VA0ioyCjs /tmp/tmp.lH4QLau9y0
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GD2sJked1G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6KBfYqLqhG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GD2sJked1G
++ cat /tmp/tmp.6KBfYqLqhG
++ rm /tmp/tmp.GD2sJked1G /tmp/tmp.6KBfYqLqhG
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HnvvhdXE7p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fjRhcebJHH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HnvvhdXE7p
++ cat /tmp/tmp.fjRhcebJHH
++ rm /tmp/tmp.HnvvhdXE7p /tmp/tmp.fjRhcebJHH
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1qAqcsRiHJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LvF4NJUCGd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1qAqcsRiHJ
++ cat /tmp/tmp.LvF4NJUCGd
++ rm /tmp/tmp.1qAqcsRiHJ /tmp/tmp.LvF4NJUCGd
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:07:58.692615 1290194 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003275176625000&timeoutSeconds=527&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:01.334587 1290599 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003280095578000&timeoutSeconds=541&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:03.637832 1290809 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003280095578000&timeoutSeconds=573&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:05.895811 1291010 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003285095773000&timeoutSeconds=348&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:08.156773 1291201 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003285095773000&timeoutSeconds=392&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:10.518437 1291409 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003290095831000&timeoutSeconds=441&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:12.848680 1291535 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003292710334000&timeoutSeconds=505&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:15.193297 1291750 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003292810081000&timeoutSeconds=515&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:17.479051 1292039 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003296305556000&timeoutSeconds=446&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:08:19.798053 1292230 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781003296305556000&timeoutSeconds=477&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6Iu1RpWQnz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fehXpbxHT6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6Iu1RpWQnz
++ cat /tmp/tmp.fehXpbxHT6
++ rm /tmp/tmp.6Iu1RpWQnz /tmp/tmp.fehXpbxHT6
++ return 0
+ pod_name=cmctl-65f9cfcc7d-x4wwt
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sjtEEhibqu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W0LJmHTKkQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sjtEEhibqu
++ cat /tmp/tmp.W0LJmHTKkQ
++ rm /tmp/tmp.sjtEEhibqu /tmp/tmp.W0LJmHTKkQ
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.C0iUnHiJuK
++ mktemp
+ local LAST_ERR=/tmp/tmp.7DgXc03cDO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C0iUnHiJuK
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-22124/some-name-ssl
+ cat /tmp/tmp.7DgXc03cDO
+ rm /tmp/tmp.C0iUnHiJuK /tmp/tmp.7DgXc03cDO
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Au001xwOZB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ggpoT2PibJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Au001xwOZB
++ cat /tmp/tmp.ggpoT2PibJ
++ rm /tmp/tmp.Au001xwOZB /tmp/tmp.ggpoT2PibJ
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tMQm2YCY5J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QGRlgI6lpX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tMQm2YCY5J
++ cat /tmp/tmp.QGRlgI6lpX
++ rm /tmp/tmp.tMQm2YCY5J /tmp/tmp.QGRlgI6lpX
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SVd91cIijY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kl2Xvvtjjz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SVd91cIijY
++ cat /tmp/tmp.Kl2Xvvtjjz
++ rm /tmp/tmp.SVd91cIijY /tmp/tmp.Kl2Xvvtjjz
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GPcBbSm1I6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3prTfbV6bb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GPcBbSm1I6
++ cat /tmp/tmp.3prTfbV6bb
++ rm /tmp/tmp.GPcBbSm1I6 /tmp/tmp.3prTfbV6bb
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dS46h7Y6zJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oMG19yF7iR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dS46h7Y6zJ
++ cat /tmp/tmp.oMG19yF7iR
++ rm /tmp/tmp.dS46h7Y6zJ /tmp/tmp.oMG19yF7iR
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PnW1KGeY0g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sekzjqRru2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PnW1KGeY0g
++ cat /tmp/tmp.sekzjqRru2
++ rm /tmp/tmp.PnW1KGeY0g /tmp/tmp.sekzjqRru2
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xRmW5Bk43c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gs3rdGa9oA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xRmW5Bk43c
++ cat /tmp/tmp.gs3rdGa9oA
++ rm /tmp/tmp.xRmW5Bk43c /tmp/tmp.gs3rdGa9oA
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.soYCao5hSc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rABV3bwbd8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.soYCao5hSc
++ cat /tmp/tmp.rABV3bwbd8
++ rm /tmp/tmp.soYCao5hSc /tmp/tmp.rABV3bwbd8
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zTpke5ijRI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pCyfhwnaJl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zTpke5ijRI
++ cat /tmp/tmp.pCyfhwnaJl
++ rm /tmp/tmp.zTpke5ijRI /tmp/tmp.pCyfhwnaJl
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zAAS1tBZhA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.21GoHZZnWB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zAAS1tBZhA
++ cat /tmp/tmp.21GoHZZnWB
++ rm /tmp/tmp.zAAS1tBZhA /tmp/tmp.21GoHZZnWB
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IiECQ1QrOt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V2Fnlthkfk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IiECQ1QrOt
++ cat /tmp/tmp.V2Fnlthkfk
++ rm /tmp/tmp.IiECQ1QrOt /tmp/tmp.V2Fnlthkfk
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 2 statefulset some-name-rs0
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FFC4hn6VdY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZIPc6FnzQr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FFC4hn6VdY
++ cat /tmp/tmp.ZIPc6FnzQr
++ rm /tmp/tmp.FFC4hn6VdY /tmp/tmp.ZIPc6FnzQr
++ return 0
+ current_generation=2
+ [[ 2 != 2 ]]
+ compare_generation 2 statefulset some-name-cfg
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hpCFSYVKUR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WVjd3ybFEh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hpCFSYVKUR
++ cat /tmp/tmp.WVjd3ybFEh
++ rm /tmp/tmp.hpCFSYVKUR /tmp/tmp.WVjd3ybFEh
++ return 0
+ current_generation=2
+ [[ 2 != 2 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:29.902673 1324966 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003667063651000&timeoutSeconds=509&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:32.227293 1325046 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003672063682000&timeoutSeconds=311&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:34.563109 1325304 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003673323412000&timeoutSeconds=461&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:36.871425 1325538 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003675576785000&timeoutSeconds=463&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:38.668797 1325606 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003675576785000&timeoutSeconds=431&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:40.997962 1325692 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003680576853000&timeoutSeconds=502&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:43.335162 1325818 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003680576853000&timeoutSeconds=370&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:45.814380 1325980 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003685577326000&timeoutSeconds=573&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:47.619789 1326032 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003687456648000&timeoutSeconds=525&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
E0609 11:14:49.423274 1326065 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl-internal&resourceVersion=1781003687456648000&timeoutSeconds=398&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3SuTox1fRD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VC2N28WWjZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3SuTox1fRD
++ cat /tmp/tmp.VC2N28WWjZ
++ rm /tmp/tmp.3SuTox1fRD /tmp/tmp.VC2N28WWjZ
++ return 0
+ pod_name=cmctl-65f9cfcc7d-x4wwt
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b9Od8SXkon
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GhL3Qn54cN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b9Od8SXkon
++ cat /tmp/tmp.GhL3Qn54cN
++ rm /tmp/tmp.b9Od8SXkon /tmp/tmp.GhL3Qn54cN
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.aWRo0EOyj1
++ mktemp
+ local LAST_ERR=/tmp/tmp.VPYSl7oGTh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aWRo0EOyj1
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-22124/some-name-ssl-internal
+ cat /tmp/tmp.VPYSl7oGTh
+ rm /tmp/tmp.aWRo0EOyj1 /tmp/tmp.VPYSl7oGTh
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HyXLcyQ3B2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FGcwTExq2t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HyXLcyQ3B2
++ cat /tmp/tmp.FGcwTExq2t
++ rm /tmp/tmp.HyXLcyQ3B2 /tmp/tmp.FGcwTExq2t
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2hNsZTOgK6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wSmxmRyQ6G
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2hNsZTOgK6
++ cat /tmp/tmp.wSmxmRyQ6G
++ rm /tmp/tmp.2hNsZTOgK6 /tmp/tmp.wSmxmRyQ6G
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rj1KtZxH2N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1bKJqCX0hX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rj1KtZxH2N
++ cat /tmp/tmp.1bKJqCX0hX
++ rm /tmp/tmp.rj1KtZxH2N /tmp/tmp.1bKJqCX0hX
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lppGEg5Zqq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pa69EdvyLJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lppGEg5Zqq
++ cat /tmp/tmp.pa69EdvyLJ
++ rm /tmp/tmp.lppGEg5Zqq /tmp/tmp.pa69EdvyLJ
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vUv6VAPMdZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SdRBBPeprq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vUv6VAPMdZ
++ cat /tmp/tmp.SdRBBPeprq
++ rm /tmp/tmp.vUv6VAPMdZ /tmp/tmp.SdRBBPeprq
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WF2WhlXdGb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X0R5KxYfFN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WF2WhlXdGb
++ cat /tmp/tmp.X0R5KxYfFN
++ rm /tmp/tmp.WF2WhlXdGb /tmp/tmp.X0R5KxYfFN
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nv9TyGvVTl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MiU60fOw9e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nv9TyGvVTl
++ cat /tmp/tmp.MiU60fOw9e
++ rm /tmp/tmp.nv9TyGvVTl /tmp/tmp.MiU60fOw9e
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F6dsuPV2sw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4m7ju5tXul
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F6dsuPV2sw
++ cat /tmp/tmp.4m7ju5tXul
++ rm /tmp/tmp.F6dsuPV2sw /tmp/tmp.4m7ju5tXul
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VIhZallCGV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H5cudqT4LZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VIhZallCGV
++ cat /tmp/tmp.H5cudqT4LZ
++ rm /tmp/tmp.VIhZallCGV /tmp/tmp.H5cudqT4LZ
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ivjkpq9KD4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.v1RzsAa164
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ivjkpq9KD4
++ cat /tmp/tmp.v1RzsAa164
++ rm /tmp/tmp.ivjkpq9KD4 /tmp/tmp.v1RzsAa164
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.390JSIlOse
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YmtDTtWLVn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.390JSIlOse
++ cat /tmp/tmp.YmtDTtWLVn
++ rm /tmp/tmp.390JSIlOse /tmp/tmp.YmtDTtWLVn
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ubjVzSRVtI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DgIlCYQitX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ubjVzSRVtI
++ cat /tmp/tmp.DgIlCYQitX
++ rm /tmp/tmp.ubjVzSRVtI /tmp/tmp.DgIlCYQitX
++ return 0
+ current_generation=3
+ [[ 3 != 3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aiPhwFUmdW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BSPjzxr8xR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aiPhwFUmdW
++ cat /tmp/tmp.BSPjzxr8xR
++ rm /tmp/tmp.aiPhwFUmdW /tmp/tmp.BSPjzxr8xR
++ return 0
+ current_generation=3
+ [[ 3 != 3 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1212
+ local resource=service/some-name-rs0
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml
+ local new_result=/tmp/tmp.6pzO5DxTVr/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-22124", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.WwebPauoEX
++ mktemp
+ local LAST_ERR=/tmp/tmp.KRMMBxPFXT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WwebPauoEX
+ cat /tmp/tmp.KRMMBxPFXT
+ rm /tmp/tmp.WwebPauoEX /tmp/tmp.KRMMBxPFXT
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6pzO5DxTVr/service_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.33 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6pzO5DxTVr/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6pzO5DxTVr/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml /tmp/tmp.6pzO5DxTVr/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2026-06-09T11:21:11+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1212
+ local resource=service/some-name-cfg
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml
+ local new_result=/tmp/tmp.6pzO5DxTVr/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-22124", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.b7SjlbK01o
++ mktemp
+ local LAST_ERR=/tmp/tmp.veoPE6UOyS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b7SjlbK01o
+ cat /tmp/tmp.veoPE6UOyS
+ rm /tmp/tmp.b7SjlbK01o /tmp/tmp.veoPE6UOyS
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6pzO5DxTVr/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6pzO5DxTVr/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6pzO5DxTVr/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml /tmp/tmp.6pzO5DxTVr/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2026-06-09T11:21:11+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1212
+ local resource=statefulset/some-name-rs0
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml
+ local new_result=/tmp/tmp.6pzO5DxTVr/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-22124", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.0CiV0O8I2R
++ mktemp
+ local LAST_ERR=/tmp/tmp.QGjKzcwC63
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0CiV0O8I2R
+ cat /tmp/tmp.QGjKzcwC63
+ rm /tmp/tmp.0CiV0O8I2R /tmp/tmp.QGjKzcwC63
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml /tmp/tmp.6pzO5DxTVr/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-06-09T11:21:12+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1212
+ local resource=statefulset/some-name-cfg
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml
+ local new_result=/tmp/tmp.6pzO5DxTVr/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-22124", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nBJJ7vcHGw
++ mktemp
+ local LAST_ERR=/tmp/tmp.FM3kojBw1d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nBJJ7vcHGw
+ cat /tmp/tmp.FM3kojBw1d
+ rm /tmp/tmp.nBJJ7vcHGw /tmp/tmp.FM3kojBw1d
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.33 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6pzO5DxTVr/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml /tmp/tmp.6pzO5DxTVr/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-06-09T11:21:13+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'test 1.22.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.22.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.22.0"}\n    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.IBZ44AKysh
++ mktemp
+ local LAST_ERR=/tmp/tmp.MeoFIBUWnZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.22.0"}\n    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IBZ44AKysh
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.MeoFIBUWnZ
+ rm /tmp/tmp.IBZ44AKysh /tmp/tmp.MeoFIBUWnZ
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A9VT4l0Wd3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.70NEZB5Nnh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A9VT4l0Wd3
++ cat /tmp/tmp.70NEZB5Nnh
++ rm /tmp/tmp.A9VT4l0Wd3 /tmp/tmp.70NEZB5Nnh
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uvAx8g4dqG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ye2cY6iS0c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uvAx8g4dqG
++ cat /tmp/tmp.ye2cY6iS0c
++ rm /tmp/tmp.uvAx8g4dqG /tmp/tmp.ye2cY6iS0c
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eZFghIVaZ3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c6qEDrMuPW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eZFghIVaZ3
++ cat /tmp/tmp.c6qEDrMuPW
++ rm /tmp/tmp.eZFghIVaZ3 /tmp/tmp.c6qEDrMuPW
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.....................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lVUH4ORqiX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EuIPewEGvA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lVUH4ORqiX
++ cat /tmp/tmp.EuIPewEGvA
++ rm /tmp/tmp.lVUH4ORqiX /tmp/tmp.EuIPewEGvA
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zt2QoJwtNB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lQRmohQ4U0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Zt2QoJwtNB
++ cat /tmp/tmp.lQRmohQ4U0
++ rm /tmp/tmp.Zt2QoJwtNB /tmp/tmp.lQRmohQ4U0
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mRZEz2Fxbr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VYnPJTHJ8D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mRZEz2Fxbr
++ cat /tmp/tmp.VYnPJTHJ8D
++ rm /tmp/tmp.mRZEz2Fxbr /tmp/tmp.VYnPJTHJ8D
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WaKjH2khO1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PFgvEnk1Nm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WaKjH2khO1
++ cat /tmp/tmp.PFgvEnk1Nm
++ rm /tmp/tmp.WaKjH2khO1 /tmp/tmp.PFgvEnk1Nm
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bbS2KcTfBG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7REYvEfXJS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bbS2KcTfBG
++ cat /tmp/tmp.7REYvEfXJS
++ rm /tmp/tmp.bbS2KcTfBG /tmp/tmp.7REYvEfXJS
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9GtQfhMYW2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gEXOoCPPNx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9GtQfhMYW2
++ cat /tmp/tmp.gEXOoCPPNx
++ rm /tmp/tmp.9GtQfhMYW2 /tmp/tmp.gEXOoCPPNx
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ML0u1ad9e9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fH5983xTJc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ML0u1ad9e9
++ cat /tmp/tmp.fH5983xTJc
++ rm /tmp/tmp.ML0u1ad9e9 /tmp/tmp.fH5983xTJc
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 4 statefulset some-name-rs0
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PqiE4YnJxz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6CSJH8X6mQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PqiE4YnJxz
++ cat /tmp/tmp.6CSJH8X6mQ
++ rm /tmp/tmp.PqiE4YnJxz /tmp/tmp.6CSJH8X6mQ
++ return 0
+ current_generation=4
+ [[ 4 != 4 ]]
+ compare_generation 4 statefulset some-name-cfg
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OxudKSDuVP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Lqg13Z5cuw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OxudKSDuVP
++ cat /tmp/tmp.Lqg13Z5cuw
++ rm /tmp/tmp.OxudKSDuVP /tmp/tmp.Lqg13Z5cuw
++ return 0
+ current_generation=4
+ [[ 4 != 4 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:35.660784 1372433 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004331416747000&timeoutSeconds=544&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:37.991243 1372551 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004336416900000&timeoutSeconds=495&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:40.339492 1372761 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004336416900000&timeoutSeconds=352&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:42.668133 1372981 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004341416916000&timeoutSeconds=481&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:45.021723 1373100 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004344706350000&timeoutSeconds=349&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:47.367260 1373218 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004347077154000&timeoutSeconds=514&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:49.745456 1373499 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004347077154000&timeoutSeconds=593&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:52.064593 1373731 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004347077154000&timeoutSeconds=360&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:53.846598 1373810 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004352077213000&timeoutSeconds=472&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
E0609 11:25:56.130841 1373913 reflector.go:227] "Failed to watch" err="Get \"https://34.41.40.27/apis/cert-manager.io/v1/namespaces/upgrade-consistency-sharded-tls-22124/certificates?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dsome-name-ssl&resourceVersion=1781004352077213000&timeoutSeconds=584&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W8L3JVi1y5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T0eUNntgfb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W8L3JVi1y5
++ cat /tmp/tmp.T0eUNntgfb
++ rm /tmp/tmp.W8L3JVi1y5 /tmp/tmp.T0eUNntgfb
++ return 0
+ pod_name=cmctl-65f9cfcc7d-x4wwt
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2ghczEcnG8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZzV5sUQK5b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2ghczEcnG8
++ cat /tmp/tmp.ZzV5sUQK5b
++ rm /tmp/tmp.2ghczEcnG8 /tmp/tmp.ZzV5sUQK5b
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.B5dpGpeyiG
++ mktemp
+ local LAST_ERR=/tmp/tmp.c6DLfOJPW8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-x4wwt -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B5dpGpeyiG
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-22124/some-name-ssl
+ cat /tmp/tmp.c6DLfOJPW8
+ rm /tmp/tmp.B5dpGpeyiG /tmp/tmp.c6DLfOJPW8
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0rMxaJDJQt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pCXjOOWmvD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0rMxaJDJQt
++ cat /tmp/tmp.pCXjOOWmvD
++ rm /tmp/tmp.0rMxaJDJQt /tmp/tmp.pCXjOOWmvD
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GLxBCqqg8n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LAs00PwHNL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GLxBCqqg8n
++ cat /tmp/tmp.LAs00PwHNL
++ rm /tmp/tmp.GLxBCqqg8n /tmp/tmp.LAs00PwHNL
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Py9bi6ucVT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WQtTWttBm2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Py9bi6ucVT
++ cat /tmp/tmp.WQtTWttBm2
++ rm /tmp/tmp.Py9bi6ucVT /tmp/tmp.WQtTWttBm2
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KnN2wAXuKv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yMgWT2vvry
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KnN2wAXuKv
++ cat /tmp/tmp.yMgWT2vvry
++ rm /tmp/tmp.KnN2wAXuKv /tmp/tmp.yMgWT2vvry
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........................................