Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/logs/upgrade-consistency-sharded-tls.log
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.27) exceeds the supported minor version skew of +/-1
+ CLUSTER=some-name
+ main
+ create_infra upgrade-consistency-sharded-tls-6997
+ local ns=upgrade-consistency-sharded-tls-6997
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.B7gjMJqikI
++ mktemp
+ local LAST_ERR=/tmp/tmp.MSgd6L7lII
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B7gjMJqikI
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.MSgd6L7lII
+ rm /tmp/tmp.B7gjMJqikI /tmp/tmp.MSgd6L7lII
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.AmXWDtP28w
++ mktemp
+ local LAST_ERR=/tmp/tmp.jGbcyTMELE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AmXWDtP28w
+ cat /tmp/tmp.jGbcyTMELE
+ rm /tmp/tmp.AmXWDtP28w /tmp/tmp.jGbcyTMELE
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.E3w8lVunp4
++ mktemp
+ local LAST_ERR=/tmp/tmp.vABa5GbOrd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E3w8lVunp4
+ cat /tmp/tmp.vABa5GbOrd
+ rm /tmp/tmp.E3w8lVunp4 /tmp/tmp.vABa5GbOrd
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-5395 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ShPPJgME6t
++ mktemp
+ local LAST_ERR=/tmp/tmp.piJdhAyFWm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ShPPJgME6t
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.piJdhAyFWm
+ rm /tmp/tmp.ShPPJgME6t /tmp/tmp.piJdhAyFWm
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.bL7m1lRt3t
++ mktemp
+ local LAST_ERR=/tmp/tmp.S0YlBWEsYD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bL7m1lRt3t
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.S0YlBWEsYD
+ rm /tmp/tmp.bL7m1lRt3t /tmp/tmp.S0YlBWEsYD
+ return 0
+ check_crd_for_deletion PR-1598-171aada3
+ local git_tag=PR-1598-171aada3
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1598-171aada3/deploy/crd.yaml
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tTBuUZnPzI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8LPhmQM0v1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.tTBuUZnPzI
++ cat /tmp/tmp.8LPhmQM0v1
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.tTBuUZnPzI
++ cat /tmp/tmp.8LPhmQM0v1
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.tTBuUZnPzI
++ cat /tmp/tmp.8LPhmQM0v1
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.tTBuUZnPzI
++ cat /tmp/tmp.8LPhmQM0v1
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.tTBuUZnPzI /tmp/tmp.8LPhmQM0v1
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.W8bbMkMNKt
+ local LAST_OUT=/tmp/tmp.0yHuZVh2Zf
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.MNaUoPI2Ht
+ local LAST_ERR=/tmp/tmp.ZHwFee8RJ4
+ local exit_status=0
+ local exit_status=0
+ local timeout=4
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0yHuZVh2Zf
+ cat /tmp/tmp.ZHwFee8RJ4
+ rm /tmp/tmp.0yHuZVh2Zf /tmp/tmp.ZHwFee8RJ4
+ return 0
namespace "cert-manager" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "upgrade-consistency-sharded-tls-5395" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.W8bbMkMNKt
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.MNaUoPI2Ht
+ rm /tmp/tmp.W8bbMkMNKt /tmp/tmp.MNaUoPI2Ht
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.gXzhIIbCi8
++ mktemp
+ local LAST_ERR=/tmp/tmp.IN7QlSVBQk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gXzhIIbCi8
+ cat /tmp/tmp.IN7QlSVBQk
+ rm /tmp/tmp.gXzhIIbCi8 /tmp/tmp.IN7QlSVBQk
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.Tc1iR3PYAB
++ mktemp
+ local LAST_ERR=/tmp/tmp.BR0GSOL9qy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Tc1iR3PYAB
namespace/psmdb-operator created
+ cat /tmp/tmp.BR0GSOL9qy
+ rm /tmp/tmp.Tc1iR3PYAB /tmp/tmp.BR0GSOL9qy
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9eNYOu5j5J
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MuLPGYBea5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9eNYOu5j5J
++ cat /tmp/tmp.MuLPGYBea5
++ rm /tmp/tmp.9eNYOu5j5J /tmp/tmp.MuLPGYBea5
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.yzF6PNHblL
++ mktemp
+ local LAST_ERR=/tmp/tmp.ycusYdDwDF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yzF6PNHblL
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5" modified.
+ cat /tmp/tmp.ycusYdDwDF
+ rm /tmp/tmp.yzF6PNHblL /tmp/tmp.ycusYdDwDF
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.WSE1hqHNy5
++ mktemp
+ local LAST_ERR=/tmp/tmp.JAt257IRmE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WSE1hqHNy5
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.JAt257IRmE
+ rm /tmp/tmp.WSE1hqHNy5 /tmp/tmp.JAt257IRmE
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ kubectl_bin apply -n psmdb-operator -f -
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zSW6w1e15N
++ mktemp
+ local LAST_ERR=/tmp/tmp.BJZptchUmn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zSW6w1e15N
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.BJZptchUmn
+ rm /tmp/tmp.zSW6w1e15N /tmp/tmp.BJZptchUmn
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1598-171aada3") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pw9QGfTJsx
++ mktemp
+ local LAST_ERR=/tmp/tmp.mInVmOjc4y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pw9QGfTJsx
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.mInVmOjc4y
+ rm /tmp/tmp.Pw9QGfTJsx /tmp/tmp.mInVmOjc4y
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XkjA4UC4a7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zoNVMs0PKP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XkjA4UC4a7
++ cat /tmp/tmp.zoNVMs0PKP
++ rm /tmp/tmp.XkjA4UC4a7 /tmp/tmp.zoNVMs0PKP
++ return 0
+ wait_pod percona-server-mongodb-operator-6b5f444cb7-7hnmk
+ local pod=percona-server-mongodb-operator-6b5f444cb7-7hnmk
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6b5f444cb7-7hnmk to be ready.OK
+ create_namespace upgrade-consistency-sharded-tls-6997
+ local namespace=upgrade-consistency-sharded-tls-6997
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-6997'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-sharded-tls-6997
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-sharded-tls-6997 --ignore-not-found
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.NRTf3ghEAu
++ mktemp
+ local LAST_ERR=/tmp/tmp.UcYilx9Ze2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.wutvWrfhdl
++ mktemp
+ local LAST_ERR=/tmp/tmp.sywy0oXbWF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-sharded-tls-6997 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wutvWrfhdl
+ cat /tmp/tmp.sywy0oXbWF
+ rm /tmp/tmp.wutvWrfhdl /tmp/tmp.sywy0oXbWF
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NRTf3ghEAu
+ cat /tmp/tmp.UcYilx9Ze2
+ rm /tmp/tmp.NRTf3ghEAu /tmp/tmp.UcYilx9Ze2
+ return 0
+ return 0
+ kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-6997
++ mktemp
+ local LAST_OUT=/tmp/tmp.xE9hq8Ujgt
++ mktemp
+ local LAST_ERR=/tmp/tmp.xBFdhR2Dlo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-6997
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xE9hq8Ujgt
+ cat /tmp/tmp.xBFdhR2Dlo
+ rm /tmp/tmp.xE9hq8Ujgt /tmp/tmp.xBFdhR2Dlo
+ return 0
+ desc 'create namespace upgrade-consistency-sharded-tls-6997'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-sharded-tls-6997
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-sharded-tls-6997
++ mktemp
+ local LAST_OUT=/tmp/tmp.bKOURFYPvw
++ mktemp
+ local LAST_ERR=/tmp/tmp.IB0kENwkns
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-consistency-sharded-tls-6997
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bKOURFYPvw
namespace/upgrade-consistency-sharded-tls-6997 created
+ cat /tmp/tmp.IB0kENwkns
+ rm /tmp/tmp.bKOURFYPvw /tmp/tmp.IB0kENwkns
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pExNmG2dTJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zom5fD98Q0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pExNmG2dTJ
++ cat /tmp/tmp.zom5fD98Q0
++ rm /tmp/tmp.pExNmG2dTJ /tmp/tmp.zom5fD98Q0
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5 --namespace=upgrade-consistency-sharded-tls-6997
++ mktemp
+ local LAST_OUT=/tmp/tmp.bLooVL8GTU
++ mktemp
+ local LAST_ERR=/tmp/tmp.wKLkD5la2l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5 --namespace=upgrade-consistency-sharded-tls-6997
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bLooVL8GTU
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1598-171aada3-1-cluster5" modified.
+ cat /tmp/tmp.wKLkD5la2l
+ rm /tmp/tmp.bLooVL8GTU /tmp/tmp.wKLkD5la2l
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.hkzSWH5Z3m
++ mktemp
+ local LAST_ERR=/tmp/tmp.muaVd5e8tG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hkzSWH5Z3m
namespace/cert-manager created
+ cat /tmp/tmp.muaVd5e8tG
+ rm /tmp/tmp.hkzSWH5Z3m /tmp/tmp.muaVd5e8tG
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.IzHegg6oxL
++ mktemp
+ local LAST_ERR=/tmp/tmp.zGj9U0Pg7C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IzHegg6oxL
namespace/cert-manager labeled
+ cat /tmp/tmp.zGj9U0Pg7C
+ rm /tmp/tmp.IzHegg6oxL /tmp/tmp.zGj9U0Pg7C
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.RLAdYE4APd
++ mktemp
+ local LAST_ERR=/tmp/tmp.zeM1isIwAV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RLAdYE4APd
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.zeM1isIwAV
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.RLAdYE4APd /tmp/tmp.zeM1isIwAV
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.ifepfrqktU
++ mktemp
+ local LAST_ERR=/tmp/tmp.oMkwFcrqQn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ifepfrqktU
pod/cert-manager-bd44d64d-85mt8 condition met
pod/cert-manager-cainjector-7dcddbd8b9-wqbbs condition met
pod/cert-manager-webhook-6cc8fdfd7-55hxx condition met
+ cat /tmp/tmp.oMkwFcrqQn
+ rm /tmp/tmp.ifepfrqktU /tmp/tmp.oMkwFcrqQn
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZPesyAcWlD
++ mktemp
+ local LAST_ERR=/tmp/tmp.WmvjAOM7In
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZPesyAcWlD
secret/some-users created
+ cat /tmp/tmp.WmvjAOM7In
+ rm /tmp/tmp.ZPesyAcWlD /tmp/tmp.WmvjAOM7In
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ljkD8EgX2g
++ mktemp
+ local LAST_ERR=/tmp/tmp.Mt5PN7NF7L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ljkD8EgX2g
deployment.apps/psmdb-client created
+ cat /tmp/tmp.Mt5PN7NF7L
+ rm /tmp/tmp.ljkD8EgX2g /tmp/tmp.Mt5PN7NF7L
+ return 0
+ deploy_cmctl
+ local service_account=cmctl
+ /usr/bin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/rbac.yaml
+ yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FiEuyDT2mN
++ mktemp
+ local LAST_ERR=/tmp/tmp.MjKwL2zqo1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FiEuyDT2mN
role.rbac.authorization.k8s.io/cmctl created
serviceaccount/cmctl created
rolebinding.rbac.authorization.k8s.io/service-account-cmctl created
+ cat /tmp/tmp.MjKwL2zqo1
+ rm /tmp/tmp.FiEuyDT2mN /tmp/tmp.MjKwL2zqo1
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/cmctl.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hXeHeRglv0
++ mktemp
+ local LAST_ERR=/tmp/tmp.aLl4TwfUwL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/conf/cmctl.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hXeHeRglv0
deployment.apps/cmctl created
+ cat /tmp/tmp.aLl4TwfUwL
+ rm /tmp/tmp.hXeHeRglv0 /tmp/tmp.aLl4TwfUwL
+ return 0
+ desc 'create first PSMDB cluster 1.15.0 some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster 1.15.0 some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1598-171aada3"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ local LAST_OUT=/tmp/tmp.oQ9hStPzH3
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.03R1Vs5jgK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oQ9hStPzH3
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.03R1Vs5jgK
+ rm /tmp/tmp.oQ9hStPzH3 /tmp/tmp.03R1Vs5jgK
+ return 0
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pYPHCV5Kjt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vzc07ax9lz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pYPHCV5Kjt
++ cat /tmp/tmp.Vzc07ax9lz
++ rm /tmp/tmp.pYPHCV5Kjt /tmp/tmp.Vzc07ax9lz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nt4hn3m4Ba
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q0qZ9HbfJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nt4hn3m4Ba
++ cat /tmp/tmp.Q0qZ9HbfJ0
++ rm /tmp/tmp.Nt4hn3m4Ba /tmp/tmp.Q0qZ9HbfJ0
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.55mtozeTND
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FfDAwNsPPf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.55mtozeTND
++ cat /tmp/tmp.FfDAwNsPPf
++ rm /tmp/tmp.55mtozeTND /tmp/tmp.FfDAwNsPPf
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1ufBtjy1bD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EbrYHLZMi5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1ufBtjy1bD
++ cat /tmp/tmp.EbrYHLZMi5
++ rm /tmp/tmp.1ufBtjy1bD /tmp/tmp.EbrYHLZMi5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AUhXIdSW6i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1WsgAZpLcU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AUhXIdSW6i
++ cat /tmp/tmp.1WsgAZpLcU
++ rm /tmp/tmp.AUhXIdSW6i /tmp/tmp.1WsgAZpLcU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XEpcMS6cBw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6TEthdNMDK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XEpcMS6cBw
++ cat /tmp/tmp.6TEthdNMDK
++ rm /tmp/tmp.XEpcMS6cBw /tmp/tmp.6TEthdNMDK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xRmhyUX9G3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qOozSRupR5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xRmhyUX9G3
++ cat /tmp/tmp.qOozSRupR5
++ rm /tmp/tmp.xRmhyUX9G3 /tmp/tmp.qOozSRupR5
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 1 statefulset some-name-rs0
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.El1dpdIDEa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nOwotRy8ft
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.El1dpdIDEa
++ cat /tmp/tmp.nOwotRy8ft
++ rm /tmp/tmp.El1dpdIDEa /tmp/tmp.nOwotRy8ft
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ compare_generation 1 statefulset some-name-cfg
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GNOJE9x0Lr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B0Lmiwujl6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GNOJE9x0Lr
++ cat /tmp/tmp.B0Lmiwujl6
++ rm /tmp/tmp.GNOJE9x0Lr /tmp/tmp.B0Lmiwujl6
++ return 0
+ current_generation=1
+ [[ 1 != \1 ]]
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DDgNQdJTms
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GFHJPWM26F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DDgNQdJTms
++ cat /tmp/tmp.GFHJPWM26F
++ rm /tmp/tmp.DDgNQdJTms /tmp/tmp.GFHJPWM26F
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V5GUAHHGU3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xzhcSE3whX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V5GUAHHGU3
++ cat /tmp/tmp.xzhcSE3whX
++ rm /tmp/tmp.V5GUAHHGU3 /tmp/tmp.xzhcSE3whX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mGNCeGcfpT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NISqjrvTWh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mGNCeGcfpT
++ cat /tmp/tmp.NISqjrvTWh
++ rm /tmp/tmp.mGNCeGcfpT /tmp/tmp.NISqjrvTWh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z7QRzJ9w5l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vsG4FsAvVH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z7QRzJ9w5l
++ cat /tmp/tmp.vsG4FsAvVH
++ rm /tmp/tmp.z7QRzJ9w5l /tmp/tmp.vsG4FsAvVH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5xc5SjED4N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sNJw9s1FSV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5xc5SjED4N
++ cat /tmp/tmp.sNJw9s1FSV
++ rm /tmp/tmp.5xc5SjED4N /tmp/tmp.sNJw9s1FSV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0TSieHi9Hi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2RUfUTx7Fw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0TSieHi9Hi
++ cat /tmp/tmp.2RUfUTx7Fw
++ rm /tmp/tmp.0TSieHi9Hi /tmp/tmp.2RUfUTx7Fw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l8WEGlHadE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oj0rpfaY43
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l8WEGlHadE
++ cat /tmp/tmp.oj0rpfaY43
++ rm /tmp/tmp.l8WEGlHadE /tmp/tmp.oj0rpfaY43
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WAYg4d18fm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xBzMGzCuUx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WAYg4d18fm
++ cat /tmp/tmp.xBzMGzCuUx
++ rm /tmp/tmp.WAYg4d18fm /tmp/tmp.xBzMGzCuUx
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WzHXoY3dsu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eF2glDD3dA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WzHXoY3dsu
++ cat /tmp/tmp.eF2glDD3dA
++ rm /tmp/tmp.WzHXoY3dsu /tmp/tmp.eF2glDD3dA
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.cHvC1S8YdE
++ mktemp
+ local LAST_ERR=/tmp/tmp.ha6VOBKhFB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cHvC1S8YdE
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl
+ cat /tmp/tmp.ha6VOBKhFB
+ rm /tmp/tmp.cHvC1S8YdE /tmp/tmp.ha6VOBKhFB
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F0ZN1Yr6ia
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nEburG6Wsd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F0ZN1Yr6ia
++ cat /tmp/tmp.nEburG6Wsd
++ rm /tmp/tmp.F0ZN1Yr6ia /tmp/tmp.nEburG6Wsd
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l9aLcvfQnw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.68JDhKB8RO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l9aLcvfQnw
++ cat /tmp/tmp.68JDhKB8RO
++ rm /tmp/tmp.l9aLcvfQnw /tmp/tmp.68JDhKB8RO
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gcpC7fRHIY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ut7qnf0SXp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gcpC7fRHIY
++ cat /tmp/tmp.ut7qnf0SXp
++ rm /tmp/tmp.gcpC7fRHIY /tmp/tmp.ut7qnf0SXp
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.....................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1FBqWEEefN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZH66I1VCJs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1FBqWEEefN
++ cat /tmp/tmp.ZH66I1VCJs
++ rm /tmp/tmp.1FBqWEEefN /tmp/tmp.ZH66I1VCJs
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YGdzsseBgJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.il1CSOx0wN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YGdzsseBgJ
++ cat /tmp/tmp.il1CSOx0wN
++ rm /tmp/tmp.YGdzsseBgJ /tmp/tmp.il1CSOx0wN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KRQAIEMeX9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b0GsQZ4J1D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KRQAIEMeX9
++ cat /tmp/tmp.b0GsQZ4J1D
++ rm /tmp/tmp.KRQAIEMeX9 /tmp/tmp.b0GsQZ4J1D
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BVxDw5Ksbf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LTM1tswqh3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BVxDw5Ksbf
++ cat /tmp/tmp.LTM1tswqh3
++ rm /tmp/tmp.BVxDw5Ksbf /tmp/tmp.LTM1tswqh3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KTVpmAwfOK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CJKiR0zXTP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KTVpmAwfOK
++ cat /tmp/tmp.CJKiR0zXTP
++ rm /tmp/tmp.KTVpmAwfOK /tmp/tmp.CJKiR0zXTP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 2 statefulset some-name-rs0
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LmUpMTRbZt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oNAyZreUub
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LmUpMTRbZt
++ cat /tmp/tmp.oNAyZreUub
++ rm /tmp/tmp.LmUpMTRbZt /tmp/tmp.oNAyZreUub
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ compare_generation 2 statefulset some-name-cfg
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8o6aARPE6R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kdBmShjKBD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8o6aARPE6R
++ cat /tmp/tmp.kdBmShjKBD
++ rm /tmp/tmp.8o6aARPE6R /tmp/tmp.kdBmShjKBD
++ return 0
+ current_generation=2
+ [[ 2 != \2 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9xrljQtDOb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CRJeci1R5z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9xrljQtDOb
++ cat /tmp/tmp.CRJeci1R5z
++ rm /tmp/tmp.9xrljQtDOb /tmp/tmp.CRJeci1R5z
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X2Z6Xpbl19
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fU1tycHBdl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X2Z6Xpbl19
++ cat /tmp/tmp.fU1tycHBdl
++ rm /tmp/tmp.X2Z6Xpbl19 /tmp/tmp.fU1tycHBdl
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.jKyMPmEzrQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.QVQNeLdjUU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jKyMPmEzrQ
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl-internal
+ cat /tmp/tmp.QVQNeLdjUU
+ rm /tmp/tmp.jKyMPmEzrQ /tmp/tmp.QVQNeLdjUU
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cj45Avpn2c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SCj8busHHx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cj45Avpn2c
++ cat /tmp/tmp.SCj8busHHx
++ rm /tmp/tmp.cj45Avpn2c /tmp/tmp.SCj8busHHx
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1WEFHViD17
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rxmFholNDN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1WEFHViD17
++ cat /tmp/tmp.rxmFholNDN
++ rm /tmp/tmp.1WEFHViD17 /tmp/tmp.rxmFholNDN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tLUsri9jqm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UjDsnM7Bx1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tLUsri9jqm
++ cat /tmp/tmp.UjDsnM7Bx1
++ rm /tmp/tmp.tLUsri9jqm /tmp/tmp.UjDsnM7Bx1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yMri5LBbRp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z2EB9yA2Tw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yMri5LBbRp
++ cat /tmp/tmp.z2EB9yA2Tw
++ rm /tmp/tmp.yMri5LBbRp /tmp/tmp.z2EB9yA2Tw
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WZE3zoaGzF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YSRRO7yavA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WZE3zoaGzF
++ cat /tmp/tmp.YSRRO7yavA
++ rm /tmp/tmp.WZE3zoaGzF /tmp/tmp.YSRRO7yavA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bLedrlnSOa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JU2yxRyg70
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bLedrlnSOa
++ cat /tmp/tmp.JU2yxRyg70
++ rm /tmp/tmp.bLedrlnSOa /tmp/tmp.JU2yxRyg70
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qQBPddRJb9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pr5AYmuTRf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qQBPddRJb9
++ cat /tmp/tmp.pr5AYmuTRf
++ rm /tmp/tmp.qQBPddRJb9 /tmp/tmp.pr5AYmuTRf
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lYoEfCL9BE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kPfjM4o7g2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lYoEfCL9BE
++ cat /tmp/tmp.kPfjM4o7g2
++ rm /tmp/tmp.lYoEfCL9BE /tmp/tmp.kPfjM4o7g2
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c89t0M8tLW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.apZcPchRWh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c89t0M8tLW
++ cat /tmp/tmp.apZcPchRWh
++ rm /tmp/tmp.c89t0M8tLW /tmp/tmp.apZcPchRWh
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6abnjZKDBf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8BdC1F0MYv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6abnjZKDBf
++ cat /tmp/tmp.8BdC1F0MYv
++ rm /tmp/tmp.6abnjZKDBf /tmp/tmp.8BdC1F0MYv
++ return 0
+ current_generation=3
+ [[ 3 != \3 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1150
+ local resource=service/some-name-rs0
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1150.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1150-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.wsjuQAEGGW
++ mktemp
+ local LAST_ERR=/tmp/tmp.i9gJuCnAvh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wsjuQAEGGW
+ cat /tmp/tmp.i9gJuCnAvh
+ rm /tmp/tmp.wsjuQAEGGW /tmp/tmp.i9gJuCnAvh
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1150.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1150.yml /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1150
+ local resource=service/some-name-cfg
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1150.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1150-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SARptLELvX
++ mktemp
+ local LAST_ERR=/tmp/tmp.Yrh5cKQpMN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SARptLELvX
+ cat /tmp/tmp.Yrh5cKQpMN
+ rm /tmp/tmp.SARptLELvX /tmp/tmp.Yrh5cKQpMN
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1150.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1150.yml /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1150
+ local resource=statefulset/some-name-rs0
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EAOVnu3z3c
++ mktemp
+ local LAST_ERR=/tmp/tmp.qARtQXO08h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EAOVnu3z3c
+ cat /tmp/tmp.qARtQXO08h
+ rm /tmp/tmp.EAOVnu3z3c /tmp/tmp.qARtQXO08h
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1150
+ local resource=statefulset/some-name-cfg
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.GwnUYyPizI
++ mktemp
+ local LAST_ERR=/tmp/tmp.FHeVngDKql
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GwnUYyPizI
+ cat /tmp/tmp.FHeVngDKql
+ rm /tmp/tmp.GwnUYyPizI /tmp/tmp.FHeVngDKql
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ desc 'test 1.16.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.16.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.16.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5BfIUXfh69
++ mktemp
+ local LAST_ERR=/tmp/tmp.Y898mtydN4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.16.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5BfIUXfh69
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.Y898mtydN4
+ rm /tmp/tmp.5BfIUXfh69 /tmp/tmp.Y898mtydN4
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.knruKgGSTS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uhkbpg8TZy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.knruKgGSTS
++ cat /tmp/tmp.uhkbpg8TZy
++ rm /tmp/tmp.knruKgGSTS /tmp/tmp.uhkbpg8TZy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lt5GcZcUl3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vWQOcVcUFK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lt5GcZcUl3
++ cat /tmp/tmp.vWQOcVcUFK
++ rm /tmp/tmp.lt5GcZcUl3 /tmp/tmp.vWQOcVcUFK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l0bmFFE3pb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kaaPBJiehj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l0bmFFE3pb
++ cat /tmp/tmp.kaaPBJiehj
++ rm /tmp/tmp.l0bmFFE3pb /tmp/tmp.kaaPBJiehj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wGZtr51LWL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gVqeTp4CDL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wGZtr51LWL
++ cat /tmp/tmp.gVqeTp4CDL
++ rm /tmp/tmp.wGZtr51LWL /tmp/tmp.gVqeTp4CDL
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zYYLYDKEXD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CLAcU0G4UT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zYYLYDKEXD
++ cat /tmp/tmp.CLAcU0G4UT
++ rm /tmp/tmp.zYYLYDKEXD /tmp/tmp.CLAcU0G4UT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m0yKmMoghE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1bHzmBiRXF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m0yKmMoghE
++ cat /tmp/tmp.1bHzmBiRXF
++ rm /tmp/tmp.m0yKmMoghE /tmp/tmp.1bHzmBiRXF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9EJZs0Qjf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ni5jRK2YT1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9EJZs0Qjf0
++ cat /tmp/tmp.ni5jRK2YT1
++ rm /tmp/tmp.9EJZs0Qjf0 /tmp/tmp.ni5jRK2YT1
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 4 statefulset some-name-rs0
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qJqVcaAIAn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6i7KC3B6lm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qJqVcaAIAn
++ cat /tmp/tmp.6i7KC3B6lm
++ rm /tmp/tmp.qJqVcaAIAn /tmp/tmp.6i7KC3B6lm
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ compare_generation 4 statefulset some-name-cfg
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qGLQzVjvcZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fcKe0ngqsh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qGLQzVjvcZ
++ cat /tmp/tmp.fcKe0ngqsh
++ rm /tmp/tmp.qGLQzVjvcZ /tmp/tmp.fcKe0ngqsh
++ return 0
+ current_generation=4
+ [[ 4 != \4 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.obMB3jLRRV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zspQkAHeTQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.obMB3jLRRV
++ cat /tmp/tmp.zspQkAHeTQ
++ rm /tmp/tmp.obMB3jLRRV /tmp/tmp.zspQkAHeTQ
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RMJFQcuXGK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3PRfaYg1Cs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RMJFQcuXGK
++ cat /tmp/tmp.3PRfaYg1Cs
++ rm /tmp/tmp.RMJFQcuXGK /tmp/tmp.3PRfaYg1Cs
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.8VLJt6FwFg
++ mktemp
+ local LAST_ERR=/tmp/tmp.ns7nCE8MT6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8VLJt6FwFg
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl
+ cat /tmp/tmp.ns7nCE8MT6
+ rm /tmp/tmp.8VLJt6FwFg /tmp/tmp.ns7nCE8MT6
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nAw8luFLZt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a8BWcnw3zG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nAw8luFLZt
++ cat /tmp/tmp.a8BWcnw3zG
++ rm /tmp/tmp.nAw8luFLZt /tmp/tmp.a8BWcnw3zG
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J8Re6iacRC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aJweTiKGVX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J8Re6iacRC
++ cat /tmp/tmp.aJweTiKGVX
++ rm /tmp/tmp.J8Re6iacRC /tmp/tmp.aJweTiKGVX
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pmsnO1Zhvf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qo8Svh8MFV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pmsnO1Zhvf
++ cat /tmp/tmp.qo8Svh8MFV
++ rm /tmp/tmp.pmsnO1Zhvf /tmp/tmp.qo8Svh8MFV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G4PxmHeyFP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uZ76myrJal
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G4PxmHeyFP
++ cat /tmp/tmp.uZ76myrJal
++ rm /tmp/tmp.G4PxmHeyFP /tmp/tmp.uZ76myrJal
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dUJ6MVHaLs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aFZyax6bmF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dUJ6MVHaLs
++ cat /tmp/tmp.aFZyax6bmF
++ rm /tmp/tmp.dUJ6MVHaLs /tmp/tmp.aFZyax6bmF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.opEKkHfFhv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xneiDst8ku
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.opEKkHfFhv
++ cat /tmp/tmp.xneiDst8ku
++ rm /tmp/tmp.opEKkHfFhv /tmp/tmp.xneiDst8ku
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vp03ESUG2L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sJxhgYtWBI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vp03ESUG2L
++ cat /tmp/tmp.sJxhgYtWBI
++ rm /tmp/tmp.vp03ESUG2L /tmp/tmp.sJxhgYtWBI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1hlP6eP3yX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OdB8Zu03fP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1hlP6eP3yX
++ cat /tmp/tmp.OdB8Zu03fP
++ rm /tmp/tmp.1hlP6eP3yX /tmp/tmp.OdB8Zu03fP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 5 statefulset some-name-rs0
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sJk1XqRcgH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SojLf9TaE0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sJk1XqRcgH
++ cat /tmp/tmp.SojLf9TaE0
++ rm /tmp/tmp.sJk1XqRcgH /tmp/tmp.SojLf9TaE0
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ compare_generation 5 statefulset some-name-cfg
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JmIrkLMRNb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CZAT1lypkL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JmIrkLMRNb
++ cat /tmp/tmp.CZAT1lypkL
++ rm /tmp/tmp.JmIrkLMRNb /tmp/tmp.CZAT1lypkL
++ return 0
+ current_generation=5
+ [[ 5 != \5 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EbLogubXXE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.21bNhAUCy0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EbLogubXXE
++ cat /tmp/tmp.21bNhAUCy0
++ rm /tmp/tmp.EbLogubXXE /tmp/tmp.21bNhAUCy0
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VgJVihUpeJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bqnLjEPoiY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VgJVihUpeJ
++ cat /tmp/tmp.bqnLjEPoiY
++ rm /tmp/tmp.VgJVihUpeJ /tmp/tmp.bqnLjEPoiY
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.5D7qmZnQSP
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ijh0ZodNqW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5D7qmZnQSP
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl-internal
+ cat /tmp/tmp.Ijh0ZodNqW
+ rm /tmp/tmp.5D7qmZnQSP /tmp/tmp.Ijh0ZodNqW
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2773bX3fRD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JwPunZeuAf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2773bX3fRD
++ cat /tmp/tmp.JwPunZeuAf
++ rm /tmp/tmp.2773bX3fRD /tmp/tmp.JwPunZeuAf
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1UZ60Xuf9g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bvOCiwhlMc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1UZ60Xuf9g
++ cat /tmp/tmp.bvOCiwhlMc
++ rm /tmp/tmp.1UZ60Xuf9g /tmp/tmp.bvOCiwhlMc
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YCFkW8fisM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3OhIqZ95xU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YCFkW8fisM
++ cat /tmp/tmp.3OhIqZ95xU
++ rm /tmp/tmp.YCFkW8fisM /tmp/tmp.3OhIqZ95xU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......................................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QmTZt9nb0q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PQD2PyZaxS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QmTZt9nb0q
++ cat /tmp/tmp.PQD2PyZaxS
++ rm /tmp/tmp.QmTZt9nb0q /tmp/tmp.PQD2PyZaxS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k1WluVupwd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BAZ2q1PdPh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k1WluVupwd
++ cat /tmp/tmp.BAZ2q1PdPh
++ rm /tmp/tmp.k1WluVupwd /tmp/tmp.BAZ2q1PdPh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5IPr8ZLfSL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wGfA4rZbv9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5IPr8ZLfSL
++ cat /tmp/tmp.wGfA4rZbv9
++ rm /tmp/tmp.5IPr8ZLfSL /tmp/tmp.wGfA4rZbv9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YDOXB6fzeY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dc3fIa2nv5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YDOXB6fzeY
++ cat /tmp/tmp.Dc3fIa2nv5
++ rm /tmp/tmp.YDOXB6fzeY /tmp/tmp.Dc3fIa2nv5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RSdOkdWOh8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VtYxmgMxRe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RSdOkdWOh8
++ cat /tmp/tmp.VtYxmgMxRe
++ rm /tmp/tmp.RSdOkdWOh8 /tmp/tmp.VtYxmgMxRe
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L72kbgT8Q9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KcMmIYX6fP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L72kbgT8Q9
++ cat /tmp/tmp.KcMmIYX6fP
++ rm /tmp/tmp.L72kbgT8Q9 /tmp/tmp.KcMmIYX6fP
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kr39ufuW9K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WzuIXsON23
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kr39ufuW9K
++ cat /tmp/tmp.WzuIXsON23
++ rm /tmp/tmp.kr39ufuW9K /tmp/tmp.WzuIXsON23
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1160
+ local resource=service/some-name-rs0
+ local postfix=-1160
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1160.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1160-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
++ mktemp
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.sa4yLx8KJ8
++ mktemp
+ local LAST_ERR=/tmp/tmp.glYHCpRNQX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sa4yLx8KJ8
+ cat /tmp/tmp.glYHCpRNQX
+ rm /tmp/tmp.sa4yLx8KJ8 /tmp/tmp.glYHCpRNQX
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1160.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1160.yml /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1160
+ local resource=service/some-name-cfg
+ local postfix=-1160
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1160.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1160-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.FDMlLhXeUY
++ mktemp
+ local LAST_ERR=/tmp/tmp.VGy0ngvVu4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FDMlLhXeUY
+ cat /tmp/tmp.VGy0ngvVu4
+ rm /tmp/tmp.FDMlLhXeUY /tmp/tmp.VGy0ngvVu4
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1160.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1160.yml /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1160
+ local resource=statefulset/some-name-rs0
+ local postfix=-1160
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.CQWrXnxpLQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kVutOEJdZL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CQWrXnxpLQ
+ cat /tmp/tmp.kVutOEJdZL
+ rm /tmp/tmp.CQWrXnxpLQ /tmp/tmp.kVutOEJdZL
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1160
+ local resource=statefulset/some-name-cfg
+ local postfix=-1160
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.wQ6nZ3Ce5j
++ mktemp
+ local LAST_ERR=/tmp/tmp.8GjdKPKawn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wQ6nZ3Ce5j
+ cat /tmp/tmp.8GjdKPKawn
+ rm /tmp/tmp.wQ6nZ3Ce5j /tmp/tmp.8GjdKPKawn
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ desc 'test 1.16.1'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.16.1
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.16.1"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4Ign7AmwBF
++ mktemp
+ local LAST_ERR=/tmp/tmp.PHWhGkKlzr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.16.1"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4Ign7AmwBF
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.PHWhGkKlzr
+ rm /tmp/tmp.4Ign7AmwBF /tmp/tmp.PHWhGkKlzr
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NISoYomMkH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8oMVGAiSkY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NISoYomMkH
++ cat /tmp/tmp.8oMVGAiSkY
++ rm /tmp/tmp.NISoYomMkH /tmp/tmp.8oMVGAiSkY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K2ulPQQ3kI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AFgMDWYcY1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K2ulPQQ3kI
++ cat /tmp/tmp.AFgMDWYcY1
++ rm /tmp/tmp.K2ulPQQ3kI /tmp/tmp.AFgMDWYcY1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Gx6ezFjya
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TXkJzpIAF0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7Gx6ezFjya
++ cat /tmp/tmp.TXkJzpIAF0
++ rm /tmp/tmp.7Gx6ezFjya /tmp/tmp.TXkJzpIAF0
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.47VU15NU2v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T9Da4ZiMSo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.47VU15NU2v
++ cat /tmp/tmp.T9Da4ZiMSo
++ rm /tmp/tmp.47VU15NU2v /tmp/tmp.T9Da4ZiMSo
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3B8sAPn6Gw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q1pSDSND6N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3B8sAPn6Gw
++ cat /tmp/tmp.q1pSDSND6N
++ rm /tmp/tmp.3B8sAPn6Gw /tmp/tmp.q1pSDSND6N
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LKGgRSQAmh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7prHvaqZrY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LKGgRSQAmh
++ cat /tmp/tmp.7prHvaqZrY
++ rm /tmp/tmp.LKGgRSQAmh /tmp/tmp.7prHvaqZrY
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pZLBEbfL7k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7bv1OPjM1o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pZLBEbfL7k
++ cat /tmp/tmp.7bv1OPjM1o
++ rm /tmp/tmp.pZLBEbfL7k /tmp/tmp.7bv1OPjM1o
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lWa1MPQg12
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZeVGS3MeZC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lWa1MPQg12
++ cat /tmp/tmp.ZeVGS3MeZC
++ rm /tmp/tmp.lWa1MPQg12 /tmp/tmp.ZeVGS3MeZC
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.glghqipdUV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KrNSngddqw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.glghqipdUV
++ cat /tmp/tmp.KrNSngddqw
++ rm /tmp/tmp.glghqipdUV /tmp/tmp.KrNSngddqw
++ return 0
+ current_generation=6
+ [[ 6 != \6 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L5EkBTRpyy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zZtCOD96iy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L5EkBTRpyy
++ cat /tmp/tmp.zZtCOD96iy
++ rm /tmp/tmp.L5EkBTRpyy /tmp/tmp.zZtCOD96iy
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hC5zDjenhj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wd3NFmcTca
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hC5zDjenhj
++ cat /tmp/tmp.wd3NFmcTca
++ rm /tmp/tmp.hC5zDjenhj /tmp/tmp.wd3NFmcTca
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.UTLmRrbtvX
++ mktemp
+ local LAST_ERR=/tmp/tmp.KeKZlmBA12
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UTLmRrbtvX
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl
+ cat /tmp/tmp.KeKZlmBA12
+ rm /tmp/tmp.UTLmRrbtvX /tmp/tmp.KeKZlmBA12
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0qJvKSmPXW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RDGpCmWQbW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0qJvKSmPXW
++ cat /tmp/tmp.RDGpCmWQbW
++ rm /tmp/tmp.0qJvKSmPXW /tmp/tmp.RDGpCmWQbW
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KGVbUgaro6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K1g6Ehv5gA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KGVbUgaro6
++ cat /tmp/tmp.K1g6Ehv5gA
++ rm /tmp/tmp.KGVbUgaro6 /tmp/tmp.K1g6Ehv5gA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EnUtE7nCoU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O57W9xLiZz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EnUtE7nCoU
++ cat /tmp/tmp.O57W9xLiZz
++ rm /tmp/tmp.EnUtE7nCoU /tmp/tmp.O57W9xLiZz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.......................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dYLbDgNcUT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AAATpscBbe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dYLbDgNcUT
++ cat /tmp/tmp.AAATpscBbe
++ rm /tmp/tmp.dYLbDgNcUT /tmp/tmp.AAATpscBbe
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tj43dnY95H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pmXmPsj1cq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tj43dnY95H
++ cat /tmp/tmp.pmXmPsj1cq
++ rm /tmp/tmp.Tj43dnY95H /tmp/tmp.pmXmPsj1cq
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p8eKVlM3p4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bWrjMcfb3A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p8eKVlM3p4
++ cat /tmp/tmp.bWrjMcfb3A
++ rm /tmp/tmp.p8eKVlM3p4 /tmp/tmp.bWrjMcfb3A
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UhyLUMk07N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rQTsenfIGy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UhyLUMk07N
++ cat /tmp/tmp.rQTsenfIGy
++ rm /tmp/tmp.UhyLUMk07N /tmp/tmp.rQTsenfIGy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4gJJjXutfX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9RtVPkaKzY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4gJJjXutfX
++ cat /tmp/tmp.9RtVPkaKzY
++ rm /tmp/tmp.4gJJjXutfX /tmp/tmp.9RtVPkaKzY
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 7 statefulset some-name-rs0
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.des9z2kIXy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7mYYfEDggs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.des9z2kIXy
++ cat /tmp/tmp.7mYYfEDggs
++ rm /tmp/tmp.des9z2kIXy /tmp/tmp.7mYYfEDggs
++ return 0
+ current_generation=7
+ [[ 7 != \7 ]]
+ compare_generation 7 statefulset some-name-cfg
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0wdLi3MBZH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m4K0J1PiZH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0wdLi3MBZH
++ cat /tmp/tmp.m4K0J1PiZH
++ rm /tmp/tmp.0wdLi3MBZH /tmp/tmp.m4K0J1PiZH
++ return 0
+ current_generation=7
+ [[ 7 != \7 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k7aOPhIkIK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NemX4P4jz0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k7aOPhIkIK
++ cat /tmp/tmp.NemX4P4jz0
++ rm /tmp/tmp.k7aOPhIkIK /tmp/tmp.NemX4P4jz0
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8hzHMlm0rq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jBxRzUwdj7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8hzHMlm0rq
++ cat /tmp/tmp.jBxRzUwdj7
++ rm /tmp/tmp.8hzHMlm0rq /tmp/tmp.jBxRzUwdj7
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.umYtv4MjO2
++ mktemp
+ local LAST_ERR=/tmp/tmp.CYpjo3UdLD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.umYtv4MjO2
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl-internal
+ cat /tmp/tmp.CYpjo3UdLD
+ rm /tmp/tmp.umYtv4MjO2 /tmp/tmp.CYpjo3UdLD
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GANHEy4FcB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lk38Rl1LHF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GANHEy4FcB
++ cat /tmp/tmp.lk38Rl1LHF
++ rm /tmp/tmp.GANHEy4FcB /tmp/tmp.lk38Rl1LHF
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LGAokTjGHe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mEwBCE7cQ5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LGAokTjGHe
++ cat /tmp/tmp.mEwBCE7cQ5
++ rm /tmp/tmp.LGAokTjGHe /tmp/tmp.mEwBCE7cQ5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WwAfsmvivi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4qr7TXXhoH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WwAfsmvivi
++ cat /tmp/tmp.4qr7TXXhoH
++ rm /tmp/tmp.WwAfsmvivi /tmp/tmp.4qr7TXXhoH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cA2x0gavfB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wl3ZNWW8ZD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cA2x0gavfB
++ cat /tmp/tmp.wl3ZNWW8ZD
++ rm /tmp/tmp.cA2x0gavfB /tmp/tmp.wl3ZNWW8ZD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cb7CAaGpHL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZbaIxANh0Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cb7CAaGpHL
++ cat /tmp/tmp.ZbaIxANh0Y
++ rm /tmp/tmp.cb7CAaGpHL /tmp/tmp.ZbaIxANh0Y
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H2Pnf6S3S3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3rPecyVqg5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H2Pnf6S3S3
++ cat /tmp/tmp.3rPecyVqg5
++ rm /tmp/tmp.H2Pnf6S3S3 /tmp/tmp.3rPecyVqg5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sqX9RbTPvv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hOLM3sLnbQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sqX9RbTPvv
++ cat /tmp/tmp.hOLM3sLnbQ
++ rm /tmp/tmp.sqX9RbTPvv /tmp/tmp.hOLM3sLnbQ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C62Rmsfhwy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1n2pupqrYo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C62Rmsfhwy
++ cat /tmp/tmp.1n2pupqrYo
++ rm /tmp/tmp.C62Rmsfhwy /tmp/tmp.1n2pupqrYo
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 8 statefulset some-name-rs0
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0A1foWJMcn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8wLmPB3liW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0A1foWJMcn
++ cat /tmp/tmp.8wLmPB3liW
++ rm /tmp/tmp.0A1foWJMcn /tmp/tmp.8wLmPB3liW
++ return 0
+ current_generation=8
+ [[ 8 != \8 ]]
+ compare_generation 8 statefulset some-name-cfg
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GjlOsO24qe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NWUyNyks6n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GjlOsO24qe
++ cat /tmp/tmp.NWUyNyks6n
++ rm /tmp/tmp.GjlOsO24qe /tmp/tmp.NWUyNyks6n
++ return 0
+ current_generation=8
+ [[ 8 != \8 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1161
+ local resource=service/some-name-rs0
+ local postfix=-1161
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1161.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1161-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
++ mktemp
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.KTS6m9Z36k
++ mktemp
+ local LAST_ERR=/tmp/tmp.OFfzBkWjB9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KTS6m9Z36k
+ cat /tmp/tmp.OFfzBkWjB9
+ rm /tmp/tmp.KTS6m9Z36k /tmp/tmp.OFfzBkWjB9
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1161.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1161.yml /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1161
+ local resource=service/some-name-cfg
+ local postfix=-1161
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1161.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1161-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.9cdoKXKISi
++ mktemp
+ local LAST_ERR=/tmp/tmp.r1zNdRtiTI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9cdoKXKISi
+ cat /tmp/tmp.r1zNdRtiTI
+ rm /tmp/tmp.9cdoKXKISi /tmp/tmp.r1zNdRtiTI
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1161.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1161.yml /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1161
+ local resource=statefulset/some-name-rs0
+ local postfix=-1161
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1161.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1161-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.FRmEOvykQ5
++ mktemp
+ local LAST_ERR=/tmp/tmp.jOxvXmAxOv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FRmEOvykQ5
+ cat /tmp/tmp.jOxvXmAxOv
+ rm /tmp/tmp.FRmEOvykQ5 /tmp/tmp.jOxvXmAxOv
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1161.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1161.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1161
+ local resource=statefulset/some-name-cfg
+ local postfix=-1161
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1161.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1161-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.K6DWJ4g1gX
++ mktemp
+ local LAST_ERR=/tmp/tmp.dSgshJfVb7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K6DWJ4g1gX
+ cat /tmp/tmp.dSgshJfVb7
+ rm /tmp/tmp.K6DWJ4g1gX /tmp/tmp.dSgshJfVb7
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1161.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1161.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ desc 'test 1.17.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.17.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.17.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.6wiG2cV3Zx
++ mktemp
+ local LAST_ERR=/tmp/tmp.0OMIysd64y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch '{
        "spec": {"crVersion":"1.17.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6wiG2cV3Zx
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.0OMIysd64y
+ rm /tmp/tmp.6wiG2cV3Zx /tmp/tmp.0OMIysd64y
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SGnIZouwtF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PXMm4fD8q3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SGnIZouwtF
++ cat /tmp/tmp.PXMm4fD8q3
++ rm /tmp/tmp.SGnIZouwtF /tmp/tmp.PXMm4fD8q3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O5kjmE5Aui
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gW4IGWuxXh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O5kjmE5Aui
++ cat /tmp/tmp.gW4IGWuxXh
++ rm /tmp/tmp.O5kjmE5Aui /tmp/tmp.gW4IGWuxXh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W83KBQQpO5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LnHPQIqMOR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W83KBQQpO5
++ cat /tmp/tmp.LnHPQIqMOR
++ rm /tmp/tmp.W83KBQQpO5 /tmp/tmp.LnHPQIqMOR
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b39QXe0bVI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4rzYOzFQBM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b39QXe0bVI
++ cat /tmp/tmp.4rzYOzFQBM
++ rm /tmp/tmp.b39QXe0bVI /tmp/tmp.4rzYOzFQBM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3IbpCOranw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LDISPNffij
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3IbpCOranw
++ cat /tmp/tmp.LDISPNffij
++ rm /tmp/tmp.3IbpCOranw /tmp/tmp.LDISPNffij
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.19T5AaNtCw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nka11Dl9UE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.19T5AaNtCw
++ cat /tmp/tmp.nka11Dl9UE
++ rm /tmp/tmp.19T5AaNtCw /tmp/tmp.nka11Dl9UE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GuEsGKVllr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5ogDoWAQd7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GuEsGKVllr
++ cat /tmp/tmp.5ogDoWAQd7
++ rm /tmp/tmp.GuEsGKVllr /tmp/tmp.5ogDoWAQd7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 8 statefulset some-name-rs0
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PsiiwmHKmJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WeE6D3Goow
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PsiiwmHKmJ
++ cat /tmp/tmp.WeE6D3Goow
++ rm /tmp/tmp.PsiiwmHKmJ /tmp/tmp.WeE6D3Goow
++ return 0
+ current_generation=8
+ [[ 8 != \8 ]]
+ compare_generation 8 statefulset some-name-cfg
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UcHDmdpR3g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tyUbimh2qF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UcHDmdpR3g
++ cat /tmp/tmp.tyUbimh2qF
++ rm /tmp/tmp.UcHDmdpR3g /tmp/tmp.tyUbimh2qF
++ return 0
+ current_generation=8
+ [[ 8 != \8 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J4ebhw4SSI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ODyI5cXvGv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J4ebhw4SSI
++ cat /tmp/tmp.ODyI5cXvGv
++ rm /tmp/tmp.J4ebhw4SSI /tmp/tmp.ODyI5cXvGv
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JBgF5XdSds
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g8HuN79YGL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JBgF5XdSds
++ cat /tmp/tmp.g8HuN79YGL
++ rm /tmp/tmp.JBgF5XdSds /tmp/tmp.g8HuN79YGL
++ return 0
+ revision=4
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.a0UQR7A3R5
++ mktemp
+ local LAST_ERR=/tmp/tmp.aBHKW8483b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a0UQR7A3R5
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl
+ cat /tmp/tmp.aBHKW8483b
+ rm /tmp/tmp.a0UQR7A3R5 /tmp/tmp.aBHKW8483b
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QISnsFsrD0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DiitETiEDn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QISnsFsrD0
++ cat /tmp/tmp.DiitETiEDn
++ rm /tmp/tmp.QISnsFsrD0 /tmp/tmp.DiitETiEDn
++ return 0
+ new_revision=5
+ '[' 5 == 5 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gfoibk09B3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RNWqagbnKs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Gfoibk09B3
++ cat /tmp/tmp.RNWqagbnKs
++ rm /tmp/tmp.Gfoibk09B3 /tmp/tmp.RNWqagbnKs
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xL9iS2HWOJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z6AmCSZBoB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xL9iS2HWOJ
++ cat /tmp/tmp.Z6AmCSZBoB
++ rm /tmp/tmp.xL9iS2HWOJ /tmp/tmp.Z6AmCSZBoB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.......................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rAFJiYMkkN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hxaQn0a2IW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rAFJiYMkkN
++ cat /tmp/tmp.hxaQn0a2IW
++ rm /tmp/tmp.rAFJiYMkkN /tmp/tmp.hxaQn0a2IW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IrIklYyUlC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tmMeTfqkyy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IrIklYyUlC
++ cat /tmp/tmp.tmMeTfqkyy
++ rm /tmp/tmp.IrIklYyUlC /tmp/tmp.tmMeTfqkyy
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J38znQcSfh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KgTLn5U8MD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J38znQcSfh
++ cat /tmp/tmp.KgTLn5U8MD
++ rm /tmp/tmp.J38znQcSfh /tmp/tmp.KgTLn5U8MD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uLS4s6wfDt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TheIKAWBEt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uLS4s6wfDt
++ cat /tmp/tmp.TheIKAWBEt
++ rm /tmp/tmp.uLS4s6wfDt /tmp/tmp.TheIKAWBEt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vxbP9tXrix
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mIt57i0zTG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vxbP9tXrix
++ cat /tmp/tmp.mIt57i0zTG
++ rm /tmp/tmp.vxbP9tXrix /tmp/tmp.mIt57i0zTG
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 9 statefulset some-name-rs0
+ local generation=9
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NWRKZDEjzE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9RQOedLyyI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NWRKZDEjzE
++ cat /tmp/tmp.9RQOedLyyI
++ rm /tmp/tmp.NWRKZDEjzE /tmp/tmp.9RQOedLyyI
++ return 0
+ current_generation=9
+ [[ 9 != \9 ]]
+ compare_generation 9 statefulset some-name-cfg
+ local generation=9
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WGP5LfS2fl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S5tyXbSuyR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WGP5LfS2fl
++ cat /tmp/tmp.S5tyXbSuyR
++ rm /tmp/tmp.WGP5LfS2fl /tmp/tmp.S5tyXbSuyR
++ return 0
+ current_generation=9
+ [[ 9 != \9 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in '{1..10}'
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Iwjh3j0Yn6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p3ImDbQcCZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Iwjh3j0Yn6
++ cat /tmp/tmp.p3ImDbQcCZ
++ rm /tmp/tmp.Iwjh3j0Yn6 /tmp/tmp.p3ImDbQcCZ
++ return 0
+ pod_name=cmctl-f8bd55686-84mtn
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wqjuv1NSTo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZkrOq0Ti28
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wqjuv1NSTo
++ cat /tmp/tmp.ZkrOq0Ti28
++ rm /tmp/tmp.wqjuv1NSTo /tmp/tmp.ZkrOq0Ti28
++ return 0
+ revision=4
+ kubectl_bin exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.6gtWcgtedu
++ mktemp
+ local LAST_ERR=/tmp/tmp.4Xcu7rbnah
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec cmctl-f8bd55686-84mtn -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6gtWcgtedu
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-6997/some-name-ssl-internal
+ cat /tmp/tmp.4Xcu7rbnah
+ rm /tmp/tmp.6gtWcgtedu /tmp/tmp.4Xcu7rbnah
+ return 0
+ for i in '{1..10}'
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.67BflWDSVk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GzK5urLuRt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.67BflWDSVk
++ cat /tmp/tmp.GzK5urLuRt
++ rm /tmp/tmp.67BflWDSVk /tmp/tmp.GzK5urLuRt
++ return 0
+ new_revision=5
+ '[' 5 == 5 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qY9xtWnTzO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cJeffkjZZr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qY9xtWnTzO
++ cat /tmp/tmp.cJeffkjZZr
++ rm /tmp/tmp.qY9xtWnTzO /tmp/tmp.cJeffkjZZr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7fggNq2Vpp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fyl7JtaEXf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7fggNq2Vpp
++ cat /tmp/tmp.Fyl7JtaEXf
++ rm /tmp/tmp.7fggNq2Vpp /tmp/tmp.Fyl7JtaEXf
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eAoVT8V7lb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R1bkUVg3TM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eAoVT8V7lb
++ cat /tmp/tmp.R1bkUVg3TM
++ rm /tmp/tmp.eAoVT8V7lb /tmp/tmp.R1bkUVg3TM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZEeCuUZA7b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NRew7O4RSI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZEeCuUZA7b
++ cat /tmp/tmp.NRew7O4RSI
++ rm /tmp/tmp.ZEeCuUZA7b /tmp/tmp.NRew7O4RSI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4MHmptWKL3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pv1Ekofi4h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4MHmptWKL3
++ cat /tmp/tmp.pv1Ekofi4h
++ rm /tmp/tmp.4MHmptWKL3 /tmp/tmp.pv1Ekofi4h
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.okgCWIe0mA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TO67kmpaMT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.okgCWIe0mA
++ cat /tmp/tmp.TO67kmpaMT
++ rm /tmp/tmp.okgCWIe0mA /tmp/tmp.TO67kmpaMT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rntbWawQ64
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sICIaR21g7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rntbWawQ64
++ cat /tmp/tmp.sICIaR21g7
++ rm /tmp/tmp.rntbWawQ64 /tmp/tmp.sICIaR21g7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ compare_generation 10 statefulset some-name-rs0
+ local generation=10
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oe1pG14VDn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hrTORTRZUw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Oe1pG14VDn
++ cat /tmp/tmp.hrTORTRZUw
++ rm /tmp/tmp.Oe1pG14VDn /tmp/tmp.hrTORTRZUw
++ return 0
+ current_generation=10
+ [[ 10 != \1\0 ]]
+ compare_generation 10 statefulset some-name-cfg
+ local generation=10
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ByHssogli3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AHV0v7LV1D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ByHssogli3
++ cat /tmp/tmp.AHV0v7LV1D
++ rm /tmp/tmp.ByHssogli3 /tmp/tmp.AHV0v7LV1D
++ return 0
+ current_generation=10
+ [[ 10 != \1\0 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1170
+ local resource=service/some-name-rs0
+ local postfix=-1170
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1170.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1170-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.l4jnFhgaGi
++ mktemp
+ local LAST_ERR=/tmp/tmp.lwibDZujn6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l4jnFhgaGi
+ cat /tmp/tmp.lwibDZujn6
+ rm /tmp/tmp.l4jnFhgaGi /tmp/tmp.lwibDZujn6
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1170.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1170.yml /tmp/tmp.xuMjx075Bw/service_some-name-rs0.yml
+ compare_kubectl service/some-name-cfg -1170
+ local resource=service/some-name-cfg
+ local postfix=-1170
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1170.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1170-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.GKIYulqqxi
++ mktemp
+ local LAST_ERR=/tmp/tmp.8etERp8Guk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GKIYulqqxi
+ cat /tmp/tmp.8etERp8Guk
+ rm /tmp/tmp.GKIYulqqxi /tmp/tmp.8etERp8Guk
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1170.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1170.yml /tmp/tmp.xuMjx075Bw/service_some-name-cfg.yml
+ compare_kubectl statefulset/some-name-rs0 -1170
+ local resource=statefulset/some-name-rs0
+ local postfix=-1170
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1170.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1170-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.D8pHeLNBPj
++ mktemp
+ local LAST_ERR=/tmp/tmp.ckxfoJlXfu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D8pHeLNBPj
+ cat /tmp/tmp.ckxfoJlXfu
+ rm /tmp/tmp.D8pHeLNBPj /tmp/tmp.ckxfoJlXfu
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1170.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1170.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-rs0.yml
+ compare_kubectl statefulset/some-name-cfg -1170
+ local resource=statefulset/some-name-cfg
+ local postfix=-1170
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1170.yml
+ local new_result=/tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1170-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-6997", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.uyUwwNswQD
++ mktemp
+ local LAST_ERR=/tmp/tmp.fXSeJLLAJx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uyUwwNswQD
+ cat /tmp/tmp.fXSeJLLAJx
+ rm /tmp/tmp.uyUwwNswQD /tmp/tmp.fXSeJLLAJx
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.27 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1170.yml == */cronjob* ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1170.yml /tmp/tmp.xuMjx075Bw/statefulset_some-name-cfg.yml
+ destroy upgrade-consistency-sharded-tls-6997
+ local namespace=upgrade-consistency-sharded-tls-6997
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.U0Odf9Flos
++ mktemp
+ local LAST_ERR=/tmp/tmp.irkYhVgS3L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U0Odf9Flos
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.irkYhVgS3L
+ rm /tmp/tmp.U0Odf9Flos /tmp/tmp.irkYhVgS3L
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.VSJB9tKjO3
++ mktemp
+ local LAST_ERR=/tmp/tmp.xD8kmQtsqC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VSJB9tKjO3
+ cat /tmp/tmp.xD8kmQtsqC
+ rm /tmp/tmp.VSJB9tKjO3 /tmp/tmp.xD8kmQtsqC
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ED1i66mMe4
++ mktemp
+ local LAST_ERR=/tmp/tmp.gEcdPSnrrQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ED1i66mMe4
+ cat /tmp/tmp.gEcdPSnrrQ
+ rm /tmp/tmp.ED1i66mMe4 /tmp/tmp.gEcdPSnrrQ
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-6997 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.zyJIIVfI7e
++ mktemp
+ local LAST_ERR=/tmp/tmp.GmrWojP9ut
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zyJIIVfI7e
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.GmrWojP9ut
+ rm /tmp/tmp.zyJIIVfI7e /tmp/tmp.GmrWojP9ut
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.QXykbOaVpR
++ mktemp
+ local LAST_ERR=/tmp/tmp.UhKWfOCzYL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1598/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QXykbOaVpR
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.UhKWfOCzYL
+ rm /tmp/tmp.QXykbOaVpR /tmp/tmp.UhKWfOCzYL
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uKPiGdvEo5
++ mktemp
+ local LAST_ERR=/tmp/tmp.kYQnbqjvju
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.uKPiGdvEo5
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted
serviceaccount "cert-manager" deleted
serviceaccount "cert-manager-webhook" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ cat /tmp/tmp.kYQnbqjvju
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.uKPiGdvEo5
namespace "cert-manager" deleted
+ cat /tmp/tmp.kYQnbqjvju
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.uKPiGdvEo5
+ cat /tmp/tmp.kYQnbqjvju
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.uKPiGdvEo5
+ cat /tmp/tmp.kYQnbqjvju
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.uKPiGdvEo5 /tmp/tmp.kYQnbqjvju
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace upgrade-consistency-sharded-tls-6997
+ rm -rf /tmp/tmp.xuMjx075Bw
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ local LAST_OUT=/tmp/tmp.WvUWGgMzPu
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.QYpLwWr583
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.tA3kflEAUd
+ local LAST_ERR=/tmp/tmp.AMUo5z0XC0
+ local exit_status=0
+ local timeout=4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace upgrade-consistency-sharded-tls-6997
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator