Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/logs/upgrade-consistency-sharded-tls.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ CLUSTER=some-name
+ main
+ create_infra upgrade-consistency-sharded-tls-8065
+ local ns=upgrade-consistency-sharded-tls-8065
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.tcEi9XfyEg
++ mktemp
+ local LAST_ERR=/tmp/tmp.csbtk1chA6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tcEi9XfyEg
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.csbtk1chA6
+ rm /tmp/tmp.tcEi9XfyEg /tmp/tmp.csbtk1chA6
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.jcwL9D3eZI
++ mktemp
+ local LAST_ERR=/tmp/tmp.4DnsvdoGiL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jcwL9D3eZI
+ cat /tmp/tmp.4DnsvdoGiL
+ rm /tmp/tmp.jcwL9D3eZI /tmp/tmp.4DnsvdoGiL
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.7HugVRN09h
++ mktemp
+ local LAST_ERR=/tmp/tmp.2M8KPRSXei
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7HugVRN09h
+ cat /tmp/tmp.2M8KPRSXei
+ rm /tmp/tmp.7HugVRN09h /tmp/tmp.2M8KPRSXei
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-17119 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ioT3wvqaA6
++ mktemp
+ local LAST_ERR=/tmp/tmp.9aqMWvmPgK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ioT3wvqaA6
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.9aqMWvmPgK
+ rm /tmp/tmp.ioT3wvqaA6 /tmp/tmp.9aqMWvmPgK
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RRhmF6wpu5
++ mktemp
+ local LAST_ERR=/tmp/tmp.goPnHBai3M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RRhmF6wpu5
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.goPnHBai3M
+ rm /tmp/tmp.RRhmF6wpu5 /tmp/tmp.goPnHBai3M
+ return 0
+ check_crd_for_deletion PR-2266-f9fc5560
+ local git_tag=PR-2266-f9fc5560
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2266-f9fc5560/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NH1mr9nyWB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IY0GhlsmSQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NH1mr9nyWB
++ cat /tmp/tmp.IY0GhlsmSQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NH1mr9nyWB
++ cat /tmp/tmp.IY0GhlsmSQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NH1mr9nyWB
++ cat /tmp/tmp.IY0GhlsmSQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.NH1mr9nyWB
++ cat /tmp/tmp.IY0GhlsmSQ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.NH1mr9nyWB /tmp/tmp.IY0GhlsmSQ
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
++ mktemp
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ub3Ym7tAbL
++ mktemp
+ local LAST_OUT=/tmp/tmp.enzcat0VH2
++ mktemp
+ local LAST_ERR=/tmp/tmp.WF03kOCIHH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.8dtx3igldX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ub3Ym7tAbL
+ cat /tmp/tmp.WF03kOCIHH
+ rm /tmp/tmp.ub3Ym7tAbL /tmp/tmp.WF03kOCIHH
+ return 0
namespace "cert-manager" deleted
namespace "upgrade-consistency-sharded-tls-17119" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.enzcat0VH2
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.8dtx3igldX
+ rm /tmp/tmp.enzcat0VH2 /tmp/tmp.8dtx3igldX
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.DqoCyr5sNe
++ mktemp
+ local LAST_ERR=/tmp/tmp.x8Hk8Jg9eC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DqoCyr5sNe
+ cat /tmp/tmp.x8Hk8Jg9eC
+ rm /tmp/tmp.DqoCyr5sNe /tmp/tmp.x8Hk8Jg9eC
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6gfF8TQPMJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.9JM4JN4z03
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6gfF8TQPMJ
namespace/psmdb-operator created
+ cat /tmp/tmp.9JM4JN4z03
+ rm /tmp/tmp.6gfF8TQPMJ /tmp/tmp.9JM4JN4z03
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZHj4qy2Htp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kuBTfz6q80
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZHj4qy2Htp
++ cat /tmp/tmp.kuBTfz6q80
++ rm /tmp/tmp.ZHj4qy2Htp /tmp/tmp.kuBTfz6q80
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.nQOXJToIGe
++ mktemp
+ local LAST_ERR=/tmp/tmp.A2gBZB1MVz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nQOXJToIGe
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6" modified.
+ cat /tmp/tmp.A2gBZB1MVz
+ rm /tmp/tmp.nQOXJToIGe /tmp/tmp.A2gBZB1MVz
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2266-f9fc5560'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2266-f9fc5560
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fODxjfmm1M
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZCWDYy331Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fODxjfmm1M
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.ZCWDYy331Z
+ rm /tmp/tmp.fODxjfmm1M /tmp/tmp.ZCWDYy331Z
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kZO9ggMjHK
++ mktemp
+ local LAST_ERR=/tmp/tmp.4iyr2ijmlm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kZO9ggMjHK
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.4iyr2ijmlm
+ rm /tmp/tmp.kZO9ggMjHK /tmp/tmp.4iyr2ijmlm
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2266-f9fc5560") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7vzcOXJAbV
++ mktemp
+ local LAST_ERR=/tmp/tmp.wfD8xx9Qhb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7vzcOXJAbV
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.wfD8xx9Qhb
+ rm /tmp/tmp.7vzcOXJAbV /tmp/tmp.wfD8xx9Qhb
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4VPQ56YTcy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jD4fy3etnX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4VPQ56YTcy
++ cat /tmp/tmp.jD4fy3etnX
++ rm /tmp/tmp.4VPQ56YTcy /tmp/tmp.jD4fy3etnX
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-5f6787b77d-h974t
+ local pod=percona-server-mongodb-operator-5f6787b77d-h974t
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5f6787b77d-h974t to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i7e5txR3Lc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GmZhZbUfoZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i7e5txR3Lc
++ cat /tmp/tmp.GmZhZbUfoZ
++ rm /tmp/tmp.i7e5txR3Lc /tmp/tmp.GmZhZbUfoZ
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-5f6787b77d-h974t
++ mktemp
+ local LAST_OUT=/tmp/tmp.YbFOXUNayy
++ mktemp
+ local LAST_ERR=/tmp/tmp.YKIcZO9o3c
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-5f6787b77d-h974t
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YbFOXUNayy
+ cat /tmp/tmp.YKIcZO9o3c
+ rm /tmp/tmp.YbFOXUNayy /tmp/tmp.YKIcZO9o3c
+ return 0
2026-04-12T21:07:46.962Z	INFO	setup	Manager starting up	{"gitCommit": "f9fc5560468cd2df79998ec06ba830d1104a31a3", "gitBranch": "PR-2266-f9fc5560", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace upgrade-consistency-sharded-tls-8065
+ local namespace=upgrade-consistency-sharded-tls-8065
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-sharded-tls-8065'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-sharded-tls-8065
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-sharded-tls-8065 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.E1wDPwchn1
++ mktemp
+ local LAST_OUT=/tmp/tmp.PU2NHlSN7Z
+ local LAST_ERR=/tmp/tmp.8UNDuWSPUQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ mktemp
+ local LAST_ERR=/tmp/tmp.PmzUICgpjx
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace upgrade-consistency-sharded-tls-8065 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.E1wDPwchn1
+ cat /tmp/tmp.8UNDuWSPUQ
+ rm /tmp/tmp.E1wDPwchn1 /tmp/tmp.8UNDuWSPUQ
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PU2NHlSN7Z
+ cat /tmp/tmp.PmzUICgpjx
+ rm /tmp/tmp.PU2NHlSN7Z /tmp/tmp.PmzUICgpjx
+ return 0
+ kubectl_bin wait --for=delete namespace upgrade-consistency-sharded-tls-8065
++ mktemp
+ local LAST_OUT=/tmp/tmp.UEowQJ2viI
++ mktemp
+ local LAST_ERR=/tmp/tmp.wzaF8YYDeH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace upgrade-consistency-sharded-tls-8065
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UEowQJ2viI
+ cat /tmp/tmp.wzaF8YYDeH
+ rm /tmp/tmp.UEowQJ2viI /tmp/tmp.wzaF8YYDeH
+ return 0
+ desc 'create namespace upgrade-consistency-sharded-tls-8065'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-sharded-tls-8065
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-sharded-tls-8065
++ mktemp
+ local LAST_OUT=/tmp/tmp.2XwpWRebiV
++ mktemp
+ local LAST_ERR=/tmp/tmp.nJAssyMeCr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace upgrade-consistency-sharded-tls-8065
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2XwpWRebiV
namespace/upgrade-consistency-sharded-tls-8065 created
+ cat /tmp/tmp.nJAssyMeCr
+ rm /tmp/tmp.2XwpWRebiV /tmp/tmp.nJAssyMeCr
+ return 0
+ set_kube_ctx upgrade-consistency-sharded-tls-8065
+ local namespace=upgrade-consistency-sharded-tls-8065
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9lxFeqIDer
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O1uT4oUHD9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9lxFeqIDer
++ cat /tmp/tmp.O1uT4oUHD9
++ rm /tmp/tmp.9lxFeqIDer /tmp/tmp.O1uT4oUHD9
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6 --namespace=upgrade-consistency-sharded-tls-8065
++ mktemp
+ local LAST_OUT=/tmp/tmp.CJWoXy168m
++ mktemp
+ local LAST_ERR=/tmp/tmp.nPPZybb3DV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6 --namespace=upgrade-consistency-sharded-tls-8065
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CJWoXy168m
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2266-f9fc5560-10-cluster6" modified.
+ cat /tmp/tmp.nPPZybb3DV
+ rm /tmp/tmp.CJWoXy168m /tmp/tmp.nPPZybb3DV
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.rApdhzYnvG
++ mktemp
+ local LAST_ERR=/tmp/tmp.KHFxzj1NiK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rApdhzYnvG
namespace/cert-manager created
+ cat /tmp/tmp.KHFxzj1NiK
+ rm /tmp/tmp.rApdhzYnvG /tmp/tmp.KHFxzj1NiK
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.3quDOLKw4j
++ mktemp
+ local LAST_ERR=/tmp/tmp.ShtqagMdMf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3quDOLKw4j
namespace/cert-manager labeled
+ cat /tmp/tmp.ShtqagMdMf
+ rm /tmp/tmp.3quDOLKw4j /tmp/tmp.ShtqagMdMf
+ return 0
+ kubectl_bin apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.nE4XJFHuCH
++ mktemp
+ local LAST_ERR=/tmp/tmp.tIgJge0JMF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nE4XJFHuCH
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.tIgJge0JMF
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.nE4XJFHuCH /tmp/tmp.tIgJge0JMF
+ return 0
+ kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.YcOaojSqrP
++ mktemp
+ local LAST_ERR=/tmp/tmp.n0aDK8T4aX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YcOaojSqrP
pod/cert-manager-559d798845-4f4c7 condition met
pod/cert-manager-cainjector-64958d9c7c-w4zs4 condition met
pod/cert-manager-webhook-7fb6f99b56-vc756 condition met
+ cat /tmp/tmp.n0aDK8T4aX
+ rm /tmp/tmp.YcOaojSqrP /tmp/tmp.n0aDK8T4aX
+ return 0
+ sleep 120
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z9zXlJWozn
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZTvWiq0QCB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z9zXlJWozn
secret/some-users created
+ cat /tmp/tmp.ZTvWiq0QCB
+ rm /tmp/tmp.Z9zXlJWozn /tmp/tmp.ZTvWiq0QCB
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9zqTEmJnhU
++ mktemp
+ local LAST_ERR=/tmp/tmp.U9vFZ4gGkT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9zqTEmJnhU
deployment.apps/psmdb-client created
+ cat /tmp/tmp.U9vFZ4gGkT
+ rm /tmp/tmp.9zqTEmJnhU /tmp/tmp.U9vFZ4gGkT
+ return 0
+ deploy_cmctl
+ local service_account=cmctl
+ /usr/sbin/sed -e s/percona-server-mongodb-operator/cmctl/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/rbac.yaml
+ yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lSDyXjRZwj
++ mktemp
+ local LAST_ERR=/tmp/tmp.AqAdfhHM6Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lSDyXjRZwj
role.rbac.authorization.k8s.io/cmctl created
serviceaccount/cmctl created
rolebinding.rbac.authorization.k8s.io/service-account-cmctl created
+ cat /tmp/tmp.AqAdfhHM6Z
+ rm /tmp/tmp.lSDyXjRZwj /tmp/tmp.AqAdfhHM6Z
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/cmctl.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.npdnW20wqc
++ mktemp
+ local LAST_ERR=/tmp/tmp.bu1TSlPvpO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/conf/cmctl.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.npdnW20wqc
deployment.apps/cmctl created
+ cat /tmp/tmp.bu1TSlPvpO
+ rm /tmp/tmp.npdnW20wqc /tmp/tmp.bu1TSlPvpO
+ return 0
+ desc 'create first PSMDB cluster 1.21.2 some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster 1.21.2 some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ kubectl_bin apply -f -
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2266-f9fc5560"'
+ local LAST_OUT=/tmp/tmp.gWxsFfhNSp
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ /usr/sbin/sed -e s/NAME_SPACE/upgrade-consistency-sharded-tls-8065/g
+ local LAST_ERR=/tmp/tmp.yVNnOGRIVg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gWxsFfhNSp
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.yVNnOGRIVg
+ rm /tmp/tmp.gWxsFfhNSp /tmp/tmp.yVNnOGRIVg
+ return 0
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lz2ZuS3qkH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wj3aG84TvM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lz2ZuS3qkH
++ cat /tmp/tmp.Wj3aG84TvM
++ rm /tmp/tmp.Lz2ZuS3qkH /tmp/tmp.Wj3aG84TvM
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0ZlaFlD2WD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UCAahEZwfT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0ZlaFlD2WD
++ cat /tmp/tmp.UCAahEZwfT
++ rm /tmp/tmp.0ZlaFlD2WD /tmp/tmp.UCAahEZwfT
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BwVjvEx5tf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XxIJ5G3cCr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BwVjvEx5tf
++ cat /tmp/tmp.XxIJ5G3cCr
++ rm /tmp/tmp.BwVjvEx5tf /tmp/tmp.XxIJ5G3cCr
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZodAFTGPbh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pdrms5u5Iv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZodAFTGPbh
++ cat /tmp/tmp.pdrms5u5Iv
++ rm /tmp/tmp.ZodAFTGPbh /tmp/tmp.pdrms5u5Iv
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YGsYOE9U3B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P14GvfhACT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YGsYOE9U3B
++ cat /tmp/tmp.P14GvfhACT
++ rm /tmp/tmp.YGsYOE9U3B /tmp/tmp.P14GvfhACT
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LAWkRGyDtj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hZNvHlFHwc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LAWkRGyDtj
++ cat /tmp/tmp.hZNvHlFHwc
++ rm /tmp/tmp.LAWkRGyDtj /tmp/tmp.hZNvHlFHwc
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9EJvWR0oiT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QCv3rM66yO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9EJvWR0oiT
++ cat /tmp/tmp.QCv3rM66yO
++ rm /tmp/tmp.9EJvWR0oiT /tmp/tmp.QCv3rM66yO
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Eigw9wVju7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IjJAECVegm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Eigw9wVju7
++ cat /tmp/tmp.IjJAECVegm
++ rm /tmp/tmp.Eigw9wVju7 /tmp/tmp.IjJAECVegm
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M6o5LE6Mhc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VA1TBtw4Pe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M6o5LE6Mhc
++ cat /tmp/tmp.VA1TBtw4Pe
++ rm /tmp/tmp.M6o5LE6Mhc /tmp/tmp.VA1TBtw4Pe
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9kzmBUucGC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nr2wca18wo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9kzmBUucGC
++ cat /tmp/tmp.nr2wca18wo
++ rm /tmp/tmp.9kzmBUucGC /tmp/tmp.nr2wca18wo
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 1 statefulset some-name-rs0
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c8awcw8rvC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UJfWY1Ibtn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c8awcw8rvC
++ cat /tmp/tmp.UJfWY1Ibtn
++ rm /tmp/tmp.c8awcw8rvC /tmp/tmp.UJfWY1Ibtn
++ return 0
+ current_generation=1
+ [[ 1 != 1 ]]
+ compare_generation 1 statefulset some-name-cfg
+ local generation=1
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eQBRdwQGGm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J8EVKDvQCa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eQBRdwQGGm
++ cat /tmp/tmp.J8EVKDvQCa
++ rm /tmp/tmp.eQBRdwQGGm /tmp/tmp.J8EVKDvQCa
++ return 0
+ current_generation=1
+ [[ 1 != 1 ]]
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DTSHJv1GLR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lIvL56J5yR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DTSHJv1GLR
++ cat /tmp/tmp.lIvL56J5yR
++ rm /tmp/tmp.DTSHJv1GLR /tmp/tmp.lIvL56J5yR
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XQS5CyUbIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LpMnu3zGdD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XQS5CyUbIl
++ cat /tmp/tmp.LpMnu3zGdD
++ rm /tmp/tmp.XQS5CyUbIl /tmp/tmp.LpMnu3zGdD
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YrHirUXgye
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QiUs0yWObm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YrHirUXgye
++ cat /tmp/tmp.QiUs0yWObm
++ rm /tmp/tmp.YrHirUXgye /tmp/tmp.QiUs0yWObm
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kR7UsYLQER
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bAHfyOpMOf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kR7UsYLQER
++ cat /tmp/tmp.bAHfyOpMOf
++ rm /tmp/tmp.kR7UsYLQER /tmp/tmp.bAHfyOpMOf
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2ACmcWrKoq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6CVGqUgUGl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2ACmcWrKoq
++ cat /tmp/tmp.6CVGqUgUGl
++ rm /tmp/tmp.2ACmcWrKoq /tmp/tmp.6CVGqUgUGl
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PkNZX7eHch
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CctGs15R9c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PkNZX7eHch
++ cat /tmp/tmp.CctGs15R9c
++ rm /tmp/tmp.PkNZX7eHch /tmp/tmp.CctGs15R9c
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jPkjNgYkrR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1xkCK4YDIh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jPkjNgYkrR
++ cat /tmp/tmp.1xkCK4YDIh
++ rm /tmp/tmp.jPkjNgYkrR /tmp/tmp.1xkCK4YDIh
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0sT96a5Xzj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X51EK7fFc8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0sT96a5Xzj
++ cat /tmp/tmp.X51EK7fFc8
++ rm /tmp/tmp.0sT96a5Xzj /tmp/tmp.X51EK7fFc8
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EHMy88h8gT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BomJrAUlgs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EHMy88h8gT
++ cat /tmp/tmp.BomJrAUlgs
++ rm /tmp/tmp.EHMy88h8gT /tmp/tmp.BomJrAUlgs
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PnQcCBYxgH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XwdLeTDmic
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PnQcCBYxgH
++ cat /tmp/tmp.XwdLeTDmic
++ rm /tmp/tmp.PnQcCBYxgH /tmp/tmp.XwdLeTDmic
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z7BPLRf7wh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WDspNoGsAo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z7BPLRf7wh
++ cat /tmp/tmp.WDspNoGsAo
++ rm /tmp/tmp.z7BPLRf7wh /tmp/tmp.WDspNoGsAo
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NIyS5gAf5c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.olnvNe5vI7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NIyS5gAf5c
++ cat /tmp/tmp.olnvNe5vI7
++ rm /tmp/tmp.NIyS5gAf5c /tmp/tmp.olnvNe5vI7
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.et8ZquY2lP
++ mktemp
+ local LAST_ERR=/tmp/tmp.R57wxnvkeY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.et8ZquY2lP
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl
+ cat /tmp/tmp.R57wxnvkeY
+ rm /tmp/tmp.et8ZquY2lP /tmp/tmp.R57wxnvkeY
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IMUAB3YCoQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GkVQ0KN5NI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IMUAB3YCoQ
++ cat /tmp/tmp.GkVQ0KN5NI
++ rm /tmp/tmp.IMUAB3YCoQ /tmp/tmp.GkVQ0KN5NI
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.88Wsa16GBP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QT2u7TyP24
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.88Wsa16GBP
++ cat /tmp/tmp.QT2u7TyP24
++ rm /tmp/tmp.88Wsa16GBP /tmp/tmp.QT2u7TyP24
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ehCjk942No
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Lvr3d1vgCD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ehCjk942No
++ cat /tmp/tmp.Lvr3d1vgCD
++ rm /tmp/tmp.ehCjk942No /tmp/tmp.Lvr3d1vgCD
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Bbu4L1CLo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RuGU8OeAEY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9Bbu4L1CLo
++ cat /tmp/tmp.RuGU8OeAEY
++ rm /tmp/tmp.9Bbu4L1CLo /tmp/tmp.RuGU8OeAEY
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..............................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PDbeDxLgU0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.haUfqjYpms
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PDbeDxLgU0
++ cat /tmp/tmp.haUfqjYpms
++ rm /tmp/tmp.PDbeDxLgU0 /tmp/tmp.haUfqjYpms
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vZO28jSZcl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.58R3I8Xv6i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vZO28jSZcl
++ cat /tmp/tmp.58R3I8Xv6i
++ rm /tmp/tmp.vZO28jSZcl /tmp/tmp.58R3I8Xv6i
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WjClxCvYZy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gMTJoWjbhu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WjClxCvYZy
++ cat /tmp/tmp.gMTJoWjbhu
++ rm /tmp/tmp.WjClxCvYZy /tmp/tmp.gMTJoWjbhu
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lvdo7fGoJo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0s9bOSUV9a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lvdo7fGoJo
++ cat /tmp/tmp.0s9bOSUV9a
++ rm /tmp/tmp.lvdo7fGoJo /tmp/tmp.0s9bOSUV9a
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7Nl7m9zia4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.epn9nYSaxW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7Nl7m9zia4
++ cat /tmp/tmp.epn9nYSaxW
++ rm /tmp/tmp.7Nl7m9zia4 /tmp/tmp.epn9nYSaxW
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I65NhPC3dH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MimynO3HhX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I65NhPC3dH
++ cat /tmp/tmp.MimynO3HhX
++ rm /tmp/tmp.I65NhPC3dH /tmp/tmp.MimynO3HhX
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t2alpHQFrG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ShCpnCAb4M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t2alpHQFrG
++ cat /tmp/tmp.ShCpnCAb4M
++ rm /tmp/tmp.t2alpHQFrG /tmp/tmp.ShCpnCAb4M
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 2 statefulset some-name-rs0
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7xDjVO02Cg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AMF7Tx6Svz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7xDjVO02Cg
++ cat /tmp/tmp.AMF7Tx6Svz
++ rm /tmp/tmp.7xDjVO02Cg /tmp/tmp.AMF7Tx6Svz
++ return 0
+ current_generation=2
+ [[ 2 != 2 ]]
+ compare_generation 2 statefulset some-name-cfg
+ local generation=2
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uPRb2EqEx8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.28UKAuuZ4f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uPRb2EqEx8
++ cat /tmp/tmp.28UKAuuZ4f
++ rm /tmp/tmp.uPRb2EqEx8 /tmp/tmp.28UKAuuZ4f
++ return 0
+ current_generation=2
+ [[ 2 != 2 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0RSZImiPl8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MUiNyE5NeC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0RSZImiPl8
++ cat /tmp/tmp.MUiNyE5NeC
++ rm /tmp/tmp.0RSZImiPl8 /tmp/tmp.MUiNyE5NeC
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aPfcGIklzI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uR8XUd59DN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aPfcGIklzI
++ cat /tmp/tmp.uR8XUd59DN
++ rm /tmp/tmp.aPfcGIklzI /tmp/tmp.uR8XUd59DN
++ return 0
+ revision=1
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.QLHN7B2rAZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.ocXxODT2b8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QLHN7B2rAZ
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl-internal
+ cat /tmp/tmp.ocXxODT2b8
+ rm /tmp/tmp.QLHN7B2rAZ /tmp/tmp.ocXxODT2b8
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.spY3OJ3dJc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6uBnE4Mfzi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.spY3OJ3dJc
++ cat /tmp/tmp.6uBnE4Mfzi
++ rm /tmp/tmp.spY3OJ3dJc /tmp/tmp.6uBnE4Mfzi
++ return 0
+ new_revision=2
+ '[' 2 == 2 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fMFMqcP74z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.abHq48gGcn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fMFMqcP74z
++ cat /tmp/tmp.abHq48gGcn
++ rm /tmp/tmp.fMFMqcP74z /tmp/tmp.abHq48gGcn
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yZbtEg8iBH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ecT840g0FK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yZbtEg8iBH
++ cat /tmp/tmp.ecT840g0FK
++ rm /tmp/tmp.yZbtEg8iBH /tmp/tmp.ecT840g0FK
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BpCWbeN7Ya
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c8dNQp9Xin
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BpCWbeN7Ya
++ cat /tmp/tmp.c8dNQp9Xin
++ rm /tmp/tmp.BpCWbeN7Ya /tmp/tmp.c8dNQp9Xin
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness..........................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dt4wQdK2GB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TUKFsUKDlv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dt4wQdK2GB
++ cat /tmp/tmp.TUKFsUKDlv
++ rm /tmp/tmp.dt4wQdK2GB /tmp/tmp.TUKFsUKDlv
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ptuBRpdagQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RuI8FybQq5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ptuBRpdagQ
++ cat /tmp/tmp.RuI8FybQq5
++ rm /tmp/tmp.ptuBRpdagQ /tmp/tmp.RuI8FybQq5
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U7Lxomw1mE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S9XMb88fP5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U7Lxomw1mE
++ cat /tmp/tmp.S9XMb88fP5
++ rm /tmp/tmp.U7Lxomw1mE /tmp/tmp.S9XMb88fP5
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CQk5NdprHZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PnhJTpiE64
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CQk5NdprHZ
++ cat /tmp/tmp.PnhJTpiE64
++ rm /tmp/tmp.CQk5NdprHZ /tmp/tmp.PnhJTpiE64
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YGHYDQFU1Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RxHl9s5j10
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YGHYDQFU1Y
++ cat /tmp/tmp.RxHl9s5j10
++ rm /tmp/tmp.YGHYDQFU1Y /tmp/tmp.RxHl9s5j10
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ha49sawx56
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hUXnzM0UuM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ha49sawx56
++ cat /tmp/tmp.hUXnzM0UuM
++ rm /tmp/tmp.Ha49sawx56 /tmp/tmp.hUXnzM0UuM
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hQa55jtK5s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MYwfwpJKTh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hQa55jtK5s
++ cat /tmp/tmp.MYwfwpJKTh
++ rm /tmp/tmp.hQa55jtK5s /tmp/tmp.MYwfwpJKTh
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 3 statefulset some-name-rs0
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GNBHxVpVTN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ixvCZgRZgG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GNBHxVpVTN
++ cat /tmp/tmp.ixvCZgRZgG
++ rm /tmp/tmp.GNBHxVpVTN /tmp/tmp.ixvCZgRZgG
++ return 0
+ current_generation=3
+ [[ 3 != 3 ]]
+ compare_generation 3 statefulset some-name-cfg
+ local generation=3
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n6CG09MvJE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QORCmtKG3r
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n6CG09MvJE
++ cat /tmp/tmp.QORCmtKG3r
++ rm /tmp/tmp.n6CG09MvJE /tmp/tmp.QORCmtKG3r
++ return 0
+ current_generation=3
+ [[ 3 != 3 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1212
+ local resource=service/some-name-rs0
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vXI86Wqc56
++ mktemp
+ local LAST_ERR=/tmp/tmp.5oxUE9m5sd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vXI86Wqc56
+ cat /tmp/tmp.5oxUE9m5sd
+ rm /tmp/tmp.vXI86Wqc56 /tmp/tmp.5oxUE9m5sd
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1212.yml /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:28:42+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1212
+ local resource=service/some-name-cfg
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fHrWk1WWMw
++ mktemp
+ local LAST_ERR=/tmp/tmp.2gpRZaRzkq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fHrWk1WWMw
+ cat /tmp/tmp.2gpRZaRzkq
+ rm /tmp/tmp.fHrWk1WWMw /tmp/tmp.2gpRZaRzkq
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1212.yml /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:28:43+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1212
+ local resource=statefulset/some-name-rs0
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xJKGvmR62F
++ mktemp
+ local LAST_ERR=/tmp/tmp.FGgNGUCr8A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xJKGvmR62F
+ cat /tmp/tmp.FGgNGUCr8A
+ rm /tmp/tmp.xJKGvmR62F /tmp/tmp.FGgNGUCr8A
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.32 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1212.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:28:44+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1212
+ local resource=statefulset/some-name-cfg
+ local postfix=-1212
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.M1WnA5ALly
++ mktemp
+ local LAST_ERR=/tmp/tmp.tqYEIyBTnj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M1WnA5ALly
+ cat /tmp/tmp.tqYEIyBTnj
+ rm /tmp/tmp.M1WnA5ALly /tmp/tmp.tqYEIyBTnj
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1212.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:28:45+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'test 1.22.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.22.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.22.0"}\n    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.em5f9OuFQp
++ mktemp
+ local LAST_ERR=/tmp/tmp.VErcbsDQjm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.22.0"}\n    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.em5f9OuFQp
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.VErcbsDQjm
+ rm /tmp/tmp.em5f9OuFQp /tmp/tmp.VErcbsDQjm
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3fkvewW3Zu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SkqGGMPmfh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3fkvewW3Zu
++ cat /tmp/tmp.SkqGGMPmfh
++ rm /tmp/tmp.3fkvewW3Zu /tmp/tmp.SkqGGMPmfh
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4NT8wuN5uy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O78r8hGBck
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4NT8wuN5uy
++ cat /tmp/tmp.O78r8hGBck
++ rm /tmp/tmp.4NT8wuN5uy /tmp/tmp.O78r8hGBck
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gGTHYPv4ju
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QLAyafh5hO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gGTHYPv4ju
++ cat /tmp/tmp.QLAyafh5hO
++ rm /tmp/tmp.gGTHYPv4ju /tmp/tmp.QLAyafh5hO
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness........................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.43g7SjJlmj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NiZ7DAxsFS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.43g7SjJlmj
++ cat /tmp/tmp.NiZ7DAxsFS
++ rm /tmp/tmp.43g7SjJlmj /tmp/tmp.NiZ7DAxsFS
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p00pYBu00T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0X8gSg8r9O
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p00pYBu00T
++ cat /tmp/tmp.0X8gSg8r9O
++ rm /tmp/tmp.p00pYBu00T /tmp/tmp.0X8gSg8r9O
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dfsNKps8CT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZcOSZpVfly
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dfsNKps8CT
++ cat /tmp/tmp.ZcOSZpVfly
++ rm /tmp/tmp.dfsNKps8CT /tmp/tmp.ZcOSZpVfly
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jI00kkyqfx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JYPV4uhJ3N
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jI00kkyqfx
++ cat /tmp/tmp.JYPV4uhJ3N
++ rm /tmp/tmp.jI00kkyqfx /tmp/tmp.JYPV4uhJ3N
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VzHjRsMSfa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wmzKuVWWJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VzHjRsMSfa
++ cat /tmp/tmp.wmzKuVWWJ0
++ rm /tmp/tmp.VzHjRsMSfa /tmp/tmp.wmzKuVWWJ0
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9BwznE0SKh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y2V7wF6H8Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9BwznE0SKh
++ cat /tmp/tmp.Y2V7wF6H8Z
++ rm /tmp/tmp.9BwznE0SKh /tmp/tmp.Y2V7wF6H8Z
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PAxfjNw6mY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X45zQ61x7x
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PAxfjNw6mY
++ cat /tmp/tmp.X45zQ61x7x
++ rm /tmp/tmp.PAxfjNw6mY /tmp/tmp.X45zQ61x7x
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 4 statefulset some-name-rs0
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nU83mQyWeb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SBKiiFIXN8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nU83mQyWeb
++ cat /tmp/tmp.SBKiiFIXN8
++ rm /tmp/tmp.nU83mQyWeb /tmp/tmp.SBKiiFIXN8
++ return 0
+ current_generation=4
+ [[ 4 != 4 ]]
+ compare_generation 4 statefulset some-name-cfg
+ local generation=4
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PD3a0emVKM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.neK82FucBG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PD3a0emVKM
++ cat /tmp/tmp.neK82FucBG
++ rm /tmp/tmp.PD3a0emVKM /tmp/tmp.neK82FucBG
++ return 0
+ current_generation=4
+ [[ 4 != 4 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IcBpTUBW9u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TtHJ3hd6aK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IcBpTUBW9u
++ cat /tmp/tmp.TtHJ3hd6aK
++ rm /tmp/tmp.IcBpTUBW9u /tmp/tmp.TtHJ3hd6aK
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JyQeJRRy2F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UqMafQKInk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JyQeJRRy2F
++ cat /tmp/tmp.UqMafQKInk
++ rm /tmp/tmp.JyQeJRRy2F /tmp/tmp.UqMafQKInk
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.wiY1OUUxK7
++ mktemp
+ local LAST_ERR=/tmp/tmp.zoxaaQueOM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wiY1OUUxK7
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl
+ cat /tmp/tmp.zoxaaQueOM
+ rm /tmp/tmp.wiY1OUUxK7 /tmp/tmp.zoxaaQueOM
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIrcO2nQBp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gA08JNIqPG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kIrcO2nQBp
++ cat /tmp/tmp.gA08JNIqPG
++ rm /tmp/tmp.kIrcO2nQBp /tmp/tmp.gA08JNIqPG
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.syld9pKRHp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K36Dxsv2aG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.syld9pKRHp
++ cat /tmp/tmp.K36Dxsv2aG
++ rm /tmp/tmp.syld9pKRHp /tmp/tmp.K36Dxsv2aG
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hfqZxwkITD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pKeyzx49ww
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hfqZxwkITD
++ cat /tmp/tmp.pKeyzx49ww
++ rm /tmp/tmp.hfqZxwkITD /tmp/tmp.pKeyzx49ww
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4prvcVogWR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F0RcB5IWdk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4prvcVogWR
++ cat /tmp/tmp.F0RcB5IWdk
++ rm /tmp/tmp.4prvcVogWR /tmp/tmp.F0RcB5IWdk
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...........................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.19grgJTDiL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jHv1BrUR2s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.19grgJTDiL
++ cat /tmp/tmp.jHv1BrUR2s
++ rm /tmp/tmp.19grgJTDiL /tmp/tmp.jHv1BrUR2s
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4WTY9FjIYq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KHprbxBFrm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4WTY9FjIYq
++ cat /tmp/tmp.KHprbxBFrm
++ rm /tmp/tmp.4WTY9FjIYq /tmp/tmp.KHprbxBFrm
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KfrchaHddK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.icBDHaCun3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KfrchaHddK
++ cat /tmp/tmp.icBDHaCun3
++ rm /tmp/tmp.KfrchaHddK /tmp/tmp.icBDHaCun3
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zcgAQJqGVm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZJzxoK1wJ9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zcgAQJqGVm
++ cat /tmp/tmp.ZJzxoK1wJ9
++ rm /tmp/tmp.zcgAQJqGVm /tmp/tmp.ZJzxoK1wJ9
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rZjpt8FmXu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oR5P1qdLVl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rZjpt8FmXu
++ cat /tmp/tmp.oR5P1qdLVl
++ rm /tmp/tmp.rZjpt8FmXu /tmp/tmp.oR5P1qdLVl
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3ZJ8NmMLKA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Itp6SqMPi8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3ZJ8NmMLKA
++ cat /tmp/tmp.Itp6SqMPi8
++ rm /tmp/tmp.3ZJ8NmMLKA /tmp/tmp.Itp6SqMPi8
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BRoUHxgRcd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k8cvka8JDZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BRoUHxgRcd
++ cat /tmp/tmp.k8cvka8JDZ
++ rm /tmp/tmp.BRoUHxgRcd /tmp/tmp.k8cvka8JDZ
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 5 statefulset some-name-rs0
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9JZSBqJDrQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gj5EaCxmbK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9JZSBqJDrQ
++ cat /tmp/tmp.Gj5EaCxmbK
++ rm /tmp/tmp.9JZSBqJDrQ /tmp/tmp.Gj5EaCxmbK
++ return 0
+ current_generation=5
+ [[ 5 != 5 ]]
+ compare_generation 5 statefulset some-name-cfg
+ local generation=5
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V1S7XU4hgX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MAs4u5xVvK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V1S7XU4hgX
++ cat /tmp/tmp.MAs4u5xVvK
++ rm /tmp/tmp.V1S7XU4hgX /tmp/tmp.MAs4u5xVvK
++ return 0
+ current_generation=5
+ [[ 5 != 5 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TC4sWgeAVl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gMATT8OtOv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TC4sWgeAVl
++ cat /tmp/tmp.gMATT8OtOv
++ rm /tmp/tmp.TC4sWgeAVl /tmp/tmp.gMATT8OtOv
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BzHc9V1Fsh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rAbUosKv2y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BzHc9V1Fsh
++ cat /tmp/tmp.rAbUosKv2y
++ rm /tmp/tmp.BzHc9V1Fsh /tmp/tmp.rAbUosKv2y
++ return 0
+ revision=2
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.aNlhCAedkO
++ mktemp
+ local LAST_ERR=/tmp/tmp.UNmBGPIUb6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aNlhCAedkO
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl-internal
+ cat /tmp/tmp.UNmBGPIUb6
+ rm /tmp/tmp.aNlhCAedkO /tmp/tmp.UNmBGPIUb6
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.irjgJQvUty
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pAKlWNZgjt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.irjgJQvUty
++ cat /tmp/tmp.pAKlWNZgjt
++ rm /tmp/tmp.irjgJQvUty /tmp/tmp.pAKlWNZgjt
++ return 0
+ new_revision=3
+ '[' 3 == 3 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZTf69gOlF3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dENzOiMrsZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZTf69gOlF3
++ cat /tmp/tmp.dENzOiMrsZ
++ rm /tmp/tmp.ZTf69gOlF3 /tmp/tmp.dENzOiMrsZ
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NM5zWVr5Zt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mFzXVF06AA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NM5zWVr5Zt
++ cat /tmp/tmp.mFzXVF06AA
++ rm /tmp/tmp.NM5zWVr5Zt /tmp/tmp.mFzXVF06AA
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LNrxNSOza4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QddUnO2y3b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LNrxNSOza4
++ cat /tmp/tmp.QddUnO2y3b
++ rm /tmp/tmp.LNrxNSOza4 /tmp/tmp.QddUnO2y3b
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness...............................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.55zAVOVmBH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RMRlV6MY0n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.55zAVOVmBH
++ cat /tmp/tmp.RMRlV6MY0n
++ rm /tmp/tmp.55zAVOVmBH /tmp/tmp.RMRlV6MY0n
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ROyzfgtoit
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uFwkGk72Yd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ROyzfgtoit
++ cat /tmp/tmp.uFwkGk72Yd
++ rm /tmp/tmp.ROyzfgtoit /tmp/tmp.uFwkGk72Yd
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SS86rIk7Ss
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0cLubs622b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SS86rIk7Ss
++ cat /tmp/tmp.0cLubs622b
++ rm /tmp/tmp.SS86rIk7Ss /tmp/tmp.0cLubs622b
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rSLyS3aPV4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o0Zi7lrc4y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rSLyS3aPV4
++ cat /tmp/tmp.o0Zi7lrc4y
++ rm /tmp/tmp.rSLyS3aPV4 /tmp/tmp.o0Zi7lrc4y
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.16il86bkzD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k5H9vrCI23
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.16il86bkzD
++ cat /tmp/tmp.k5H9vrCI23
++ rm /tmp/tmp.16il86bkzD /tmp/tmp.k5H9vrCI23
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sL94oawCex
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kNqhbdSDxb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sL94oawCex
++ cat /tmp/tmp.kNqhbdSDxb
++ rm /tmp/tmp.sL94oawCex /tmp/tmp.kNqhbdSDxb
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TcwGR9ztAb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FXHj3Bo3PT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TcwGR9ztAb
++ cat /tmp/tmp.FXHj3Bo3PT
++ rm /tmp/tmp.TcwGR9ztAb /tmp/tmp.FXHj3Bo3PT
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0UpYdscWft
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J7ntDK7pOW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0UpYdscWft
++ cat /tmp/tmp.J7ntDK7pOW
++ rm /tmp/tmp.0UpYdscWft /tmp/tmp.J7ntDK7pOW
++ return 0
+ current_generation=6
+ [[ 6 != 6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E8CFRWxxXg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OUHcVjuUt9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E8CFRWxxXg
++ cat /tmp/tmp.OUHcVjuUt9
++ rm /tmp/tmp.E8CFRWxxXg /tmp/tmp.OUHcVjuUt9
++ return 0
+ current_generation=6
+ [[ 6 != 6 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1220
+ local resource=service/some-name-rs0
+ local postfix=-1220
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gg46LVMzkJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.AEmphyG1hG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gg46LVMzkJ
+ cat /tmp/tmp.AEmphyG1hG
+ rm /tmp/tmp.gg46LVMzkJ /tmp/tmp.AEmphyG1hG
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1220.yml /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:45:48+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1220
+ local resource=service/some-name-cfg
+ local postfix=-1220
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.yeLjulEpFC
++ mktemp
+ local LAST_ERR=/tmp/tmp.I2C0V6iH7d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yeLjulEpFC
+ cat /tmp/tmp.I2C0V6iH7d
+ rm /tmp/tmp.yeLjulEpFC /tmp/tmp.I2C0V6iH7d
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1220.yml /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:45:49+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1220
+ local resource=statefulset/some-name-rs0
+ local postfix=-1220
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3KP4tWdQv6
++ mktemp
+ local LAST_ERR=/tmp/tmp.e2lj9ALoWb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3KP4tWdQv6
+ cat /tmp/tmp.e2lj9ALoWb
+ rm /tmp/tmp.3KP4tWdQv6 /tmp/tmp.e2lj9ALoWb
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1220.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:45:50+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1220
+ local resource=statefulset/some-name-cfg
+ local postfix=-1220
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.OWvSqsC9Xz
++ mktemp
+ local LAST_ERR=/tmp/tmp.ddFtoXktYQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OWvSqsC9Xz
+ cat /tmp/tmp.ddFtoXktYQ
+ rm /tmp/tmp.OWvSqsC9Xz /tmp/tmp.ddFtoXktYQ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1220.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:45:50+0000] compare_kubectl: statefulset/some-name-cfg OK
+ desc 'test 1.23.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.23.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.23.0"}\n    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qN5TWLvElz
++ mktemp
+ local LAST_ERR=/tmp/tmp.5GKHD4F3Ar
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type=merge --patch $'{\n        "spec": {"crVersion":"1.23.0"}\n    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qN5TWLvElz
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.5GKHD4F3Ar
+ rm /tmp/tmp.qN5TWLvElz /tmp/tmp.5GKHD4F3Ar
+ return 0
+ sleep 20
+ desc 'check if Pod started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod started
-----------------------------------------------------------------------------------

+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KLEHlamV19
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZHOF6WH6Wr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KLEHlamV19
++ cat /tmp/tmp.ZHOF6WH6Wr
++ rm /tmp/tmp.KLEHlamV19 /tmp/tmp.ZHOF6WH6Wr
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mOQc1Hg6WH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iyJlzB5Tl5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mOQc1Hg6WH
++ cat /tmp/tmp.iyJlzB5Tl5
++ rm /tmp/tmp.mOQc1Hg6WH /tmp/tmp.iyJlzB5Tl5
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pMfpqsL0sH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ybrPdFVc83
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pMfpqsL0sH
++ cat /tmp/tmp.ybrPdFVc83
++ rm /tmp/tmp.pMfpqsL0sH /tmp/tmp.ybrPdFVc83
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uPjnBiWKYN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.r0jK2FFtAS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uPjnBiWKYN
++ cat /tmp/tmp.r0jK2FFtAS
++ rm /tmp/tmp.uPjnBiWKYN /tmp/tmp.r0jK2FFtAS
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MwsBfHFvxI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NSsUOIRVGF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MwsBfHFvxI
++ cat /tmp/tmp.NSsUOIRVGF
++ rm /tmp/tmp.MwsBfHFvxI /tmp/tmp.NSsUOIRVGF
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dkz0Kvefof
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vZ97vnfJU3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dkz0Kvefof
++ cat /tmp/tmp.vZ97vnfJU3
++ rm /tmp/tmp.dkz0Kvefof /tmp/tmp.vZ97vnfJU3
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JnTIFJVQka
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yoXmqvrIaN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JnTIFJVQka
++ cat /tmp/tmp.yoXmqvrIaN
++ rm /tmp/tmp.JnTIFJVQka /tmp/tmp.yoXmqvrIaN
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.frAvCnktCc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZ7LUtxcq8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.frAvCnktCc
++ cat /tmp/tmp.PZ7LUtxcq8
++ rm /tmp/tmp.frAvCnktCc /tmp/tmp.PZ7LUtxcq8
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jpucTLd6YO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KpaaZpHXAw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jpucTLd6YO
++ cat /tmp/tmp.KpaaZpHXAw
++ rm /tmp/tmp.jpucTLd6YO /tmp/tmp.KpaaZpHXAw
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NzGBLqIK56
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JlqMaPnU7D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NzGBLqIK56
++ cat /tmp/tmp.JlqMaPnU7D
++ rm /tmp/tmp.NzGBLqIK56 /tmp/tmp.JlqMaPnU7D
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 6 statefulset some-name-rs0
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WnH76dIebf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0vkNSZZFYI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WnH76dIebf
++ cat /tmp/tmp.0vkNSZZFYI
++ rm /tmp/tmp.WnH76dIebf /tmp/tmp.0vkNSZZFYI
++ return 0
+ current_generation=6
+ [[ 6 != 6 ]]
+ compare_generation 6 statefulset some-name-cfg
+ local generation=6
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VFtiAYNnDd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3PnifMaiA3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VFtiAYNnDd
++ cat /tmp/tmp.3PnifMaiA3
++ rm /tmp/tmp.VFtiAYNnDd /tmp/tmp.3PnifMaiA3
++ return 0
+ current_generation=6
+ [[ 6 != 6 ]]
+ renew_certificate some-name-ssl
+ certificate=some-name-ssl
+ wait_certificate some-name-ssl
+ certificate=some-name-ssl
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl --timeout=60s
certificate.cert-manager.io/some-name-ssl condition met
+ sleep 1
+ desc 'renew some-name-ssl'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WRqC7iRhtP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G0cyB4cShu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WRqC7iRhtP
++ cat /tmp/tmp.G0cyB4cShu
++ rm /tmp/tmp.WRqC7iRhtP /tmp/tmp.G0cyB4cShu
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IF2Yw2H53A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ppOx1VGhZ9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IF2Yw2H53A
++ cat /tmp/tmp.ppOx1VGhZ9
++ rm /tmp/tmp.IF2Yw2H53A /tmp/tmp.ppOx1VGhZ9
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
++ mktemp
+ local LAST_OUT=/tmp/tmp.m3fdxpdpM2
++ mktemp
+ local LAST_ERR=/tmp/tmp.BkLLMLuAFq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m3fdxpdpM2
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl
+ cat /tmp/tmp.BkLLMLuAFq
+ rm /tmp/tmp.m3fdxpdpM2 /tmp/tmp.BkLLMLuAFq
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QID6mxOLNv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7Ve8BPtHLG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QID6mxOLNv
++ cat /tmp/tmp.7Ve8BPtHLG
++ rm /tmp/tmp.QID6mxOLNv /tmp/tmp.7Ve8BPtHLG
++ return 0
+ new_revision=3
+ '[' 4 == 3 ']'
+ sleep 1
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9QugrIf5s4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TP4FMwKfyj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9QugrIf5s4
++ cat /tmp/tmp.TP4FMwKfyj
++ rm /tmp/tmp.9QugrIf5s4 /tmp/tmp.TP4FMwKfyj
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TA6kD3jMU0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PjEEGct2fc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TA6kD3jMU0
++ cat /tmp/tmp.PjEEGct2fc
++ rm /tmp/tmp.TA6kD3jMU0 /tmp/tmp.PjEEGct2fc
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4Ot0VwCjwz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q7tHzvpPRF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4Ot0VwCjwz
++ cat /tmp/tmp.Q7tHzvpPRF
++ rm /tmp/tmp.4Ot0VwCjwz /tmp/tmp.Q7tHzvpPRF
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U5eE8q6hiB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.245n67eivh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U5eE8q6hiB
++ cat /tmp/tmp.245n67eivh
++ rm /tmp/tmp.U5eE8q6hiB /tmp/tmp.245n67eivh
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.................................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rqpAQ6So6K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fHJ7e98wMG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rqpAQ6So6K
++ cat /tmp/tmp.fHJ7e98wMG
++ rm /tmp/tmp.rqpAQ6So6K /tmp/tmp.fHJ7e98wMG
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7709vfNoeM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gqIAr075Wg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7709vfNoeM
++ cat /tmp/tmp.gqIAr075Wg
++ rm /tmp/tmp.7709vfNoeM /tmp/tmp.gqIAr075Wg
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cXr9n9Xg6T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hLMvm5b9UU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cXr9n9Xg6T
++ cat /tmp/tmp.hLMvm5b9UU
++ rm /tmp/tmp.cXr9n9Xg6T /tmp/tmp.hLMvm5b9UU
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.us823jO8oC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5U8dOqBP9V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.us823jO8oC
++ cat /tmp/tmp.5U8dOqBP9V
++ rm /tmp/tmp.us823jO8oC /tmp/tmp.5U8dOqBP9V
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1fSyrPEEao
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2lTDB6tEq1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1fSyrPEEao
++ cat /tmp/tmp.2lTDB6tEq1
++ rm /tmp/tmp.1fSyrPEEao /tmp/tmp.2lTDB6tEq1
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XOb0ueQt1Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NeuIj8LLVO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XOb0ueQt1Q
++ cat /tmp/tmp.NeuIj8LLVO
++ rm /tmp/tmp.XOb0ueQt1Q /tmp/tmp.NeuIj8LLVO
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RpCOJCmDSC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cWRkTpDZ3y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RpCOJCmDSC
++ cat /tmp/tmp.cWRkTpDZ3y
++ rm /tmp/tmp.RpCOJCmDSC /tmp/tmp.cWRkTpDZ3y
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 7 statefulset some-name-rs0
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xPt0uTONPx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.igFgiZvfMa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xPt0uTONPx
++ cat /tmp/tmp.igFgiZvfMa
++ rm /tmp/tmp.xPt0uTONPx /tmp/tmp.igFgiZvfMa
++ return 0
+ current_generation=7
+ [[ 7 != 7 ]]
+ compare_generation 7 statefulset some-name-cfg
+ local generation=7
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R3Fg8oj3Gr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1bzf9R2ofm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R3Fg8oj3Gr
++ cat /tmp/tmp.1bzf9R2ofm
++ rm /tmp/tmp.R3Fg8oj3Gr /tmp/tmp.1bzf9R2ofm
++ return 0
+ current_generation=7
+ [[ 7 != 7 ]]
+ renew_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ wait_certificate some-name-ssl-internal
+ certificate=some-name-ssl-internal
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ for i in {1..10}
+ kubectl wait --for=condition=Ready certificate/some-name-ssl-internal --timeout=60s
certificate.cert-manager.io/some-name-ssl-internal condition met
+ sleep 1
+ desc 'renew some-name-ssl-internal'
+ set +o xtrace


-----------------------------------------------------------------------------------
renew some-name-ssl-internal
-----------------------------------------------------------------------------------

+ local pod_name
++ kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qKEq88aOSV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SMFNCkTeHb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qKEq88aOSV
++ cat /tmp/tmp.SMFNCkTeHb
++ rm /tmp/tmp.qKEq88aOSV /tmp/tmp.SMFNCkTeHb
++ return 0
+ pod_name=cmctl-65f9cfcc7d-qhtkc
+ local revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2zh6lqzlHM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pvLWyuHKuB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2zh6lqzlHM
++ cat /tmp/tmp.pvLWyuHKuB
++ rm /tmp/tmp.2zh6lqzlHM /tmp/tmp.pvLWyuHKuB
++ return 0
+ revision=3
+ kubectl_bin exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
++ mktemp
+ local LAST_OUT=/tmp/tmp.uQdYCKbm5u
++ mktemp
+ local LAST_ERR=/tmp/tmp.kbDnEF02pG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec cmctl-65f9cfcc7d-qhtkc -- /tmp/cmctl renew some-name-ssl-internal
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uQdYCKbm5u
Manually triggered issuance of Certificate upgrade-consistency-sharded-tls-8065/some-name-ssl-internal
+ cat /tmp/tmp.kbDnEF02pG
+ rm /tmp/tmp.uQdYCKbm5u /tmp/tmp.kbDnEF02pG
+ return 0
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.glqVlDVvxo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.05DWK7bvYq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.glqVlDVvxo
++ cat /tmp/tmp.05DWK7bvYq
++ rm /tmp/tmp.glqVlDVvxo /tmp/tmp.05DWK7bvYq
++ return 0
+ new_revision=3
+ '[' 4 == 3 ']'
+ sleep 1
+ for i in {1..10}
+ local new_revision
++ kubectl_bin get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4GxOqAtAxW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.48dombIzBh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get certificate some-name-ssl-internal -o 'jsonpath={.status.revision}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4GxOqAtAxW
++ cat /tmp/tmp.48dombIzBh
++ rm /tmp/tmp.4GxOqAtAxW /tmp/tmp.48dombIzBh
++ return 0
+ new_revision=4
+ '[' 4 == 4 ']'
+ break
+ sleep 20
+ wait_cluster
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L5Bu2HSx39
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TWDsIAfuKb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L5Bu2HSx39
++ cat /tmp/tmp.TWDsIAfuKb
++ rm /tmp/tmp.L5Bu2HSx39 /tmp/tmp.TWDsIAfuKb
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1SELeaaI4c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4wyX075W6b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1SELeaaI4c
++ cat /tmp/tmp.4wyX075W6b
++ rm /tmp/tmp.1SELeaaI4c /tmp/tmp.4wyX075W6b
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RKgEq4yleT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PKKZmSc9wn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RKgEq4yleT
++ cat /tmp/tmp.PKKZmSc9wn
++ rm /tmp/tmp.RKgEq4yleT /tmp/tmp.PKKZmSc9wn
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.................................................................................................................................................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ubK8d6DlG8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEScW5fQRw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ubK8d6DlG8
++ cat /tmp/tmp.TEScW5fQRw
++ rm /tmp/tmp.ubK8d6DlG8 /tmp/tmp.TEScW5fQRw
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lYnRh36VeA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GgJoUxXHcN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lYnRh36VeA
++ cat /tmp/tmp.GgJoUxXHcN
++ rm /tmp/tmp.lYnRh36VeA /tmp/tmp.GgJoUxXHcN
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l0Jru4Xz1y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wj1BY5ku0S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l0Jru4Xz1y
++ cat /tmp/tmp.Wj1BY5ku0S
++ rm /tmp/tmp.l0Jru4Xz1y /tmp/tmp.Wj1BY5ku0S
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ErWRnqrTrS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.od8sljJW9F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ErWRnqrTrS
++ cat /tmp/tmp.od8sljJW9F
++ rm /tmp/tmp.ErWRnqrTrS /tmp/tmp.od8sljJW9F
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HmYDjU35jB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rWIVEFW0OY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HmYDjU35jB
++ cat /tmp/tmp.rWIVEFW0OY
++ rm /tmp/tmp.HmYDjU35jB /tmp/tmp.rWIVEFW0OY
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ekgi3CTECZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hvBfF7UvWP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ekgi3CTECZ
++ cat /tmp/tmp.hvBfF7UvWP
++ rm /tmp/tmp.Ekgi3CTECZ /tmp/tmp.hvBfF7UvWP
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OnxwW8nP5Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KGOebg6Av0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OnxwW8nP5Q
++ cat /tmp/tmp.KGOebg6Av0
++ rm /tmp/tmp.OnxwW8nP5Q /tmp/tmp.KGOebg6Av0
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ compare_generation 8 statefulset some-name-rs0
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-rs0
+ local current_generation
++ kubectl_bin get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b0ECIo4ujD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.muOBpWcEet
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-rs0 -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b0ECIo4ujD
++ cat /tmp/tmp.muOBpWcEet
++ rm /tmp/tmp.b0ECIo4ujD /tmp/tmp.muOBpWcEet
++ return 0
+ current_generation=8
+ [[ 8 != 8 ]]
+ compare_generation 8 statefulset some-name-cfg
+ local generation=8
+ local resource_type=statefulset
+ local resource_name=some-name-cfg
+ local current_generation
++ kubectl_bin get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jTmm5PHPqG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ILy9CQDNEd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get statefulset some-name-cfg -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jTmm5PHPqG
++ cat /tmp/tmp.ILy9CQDNEd
++ rm /tmp/tmp.jTmm5PHPqG /tmp/tmp.ILy9CQDNEd
++ return 0
+ current_generation=8
+ [[ 8 != 8 ]]
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-rs0 -1230
+ local resource=service/some-name-rs0
+ local postfix=-1230
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1230.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1230-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7ql3vdWUZS
++ mktemp
+ local LAST_ERR=/tmp/tmp.BmKAuwc5N1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7ql3vdWUZS
+ cat /tmp/tmp.BmKAuwc5N1
+ rm /tmp/tmp.7ql3vdWUZS /tmp/tmp.BmKAuwc5N1
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1230.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-rs0-1230.yml /tmp/tmp.btY8OyXru1/service_some-name-rs0.yml
+ log 'compare_kubectl: service/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:59:27+0000] compare_kubectl: service/some-name-rs0 OK
+ compare_kubectl service/some-name-cfg -1230
+ local resource=service/some-name-cfg
+ local postfix=-1230
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1230.yml
+ local new_result=/tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1230-oc.yml ']'
+ kubectl_bin get -o yaml service/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PsfTDUIwWb
++ mktemp
+ local LAST_ERR=/tmp/tmp.GOfaqB4XGf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml service/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PsfTDUIwWb
+ cat /tmp/tmp.GOfaqB4XGf
+ rm /tmp/tmp.PsfTDUIwWb /tmp/tmp.GOfaqB4XGf
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1230.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/service_some-name-cfg-1230.yml /tmp/tmp.btY8OyXru1/service_some-name-cfg.yml
+ log 'compare_kubectl: service/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:59:27+0000] compare_kubectl: service/some-name-cfg OK
+ compare_kubectl statefulset/some-name-rs0 -1230
+ local resource=statefulset/some-name-rs0
+ local postfix=-1230
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1230.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1230-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.N0QjcldcVk
++ mktemp
+ local LAST_ERR=/tmp/tmp.9FIh53eMju
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N0QjcldcVk
+ cat /tmp/tmp.9FIh53eMju
+ rm /tmp/tmp.N0QjcldcVk /tmp/tmp.9FIh53eMju
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1230.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1230.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-04-12T21:59:28+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-cfg -1230
+ local resource=statefulset/some-name-cfg
+ local postfix=-1230
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1230.yml
+ local new_result=/tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1230-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval $'\n\t\t\tdel(.metadata.ownerReferences[].apiVersion) |\n\t\t\tdel(.metadata.managedFields) |\n\t\t\tdel(.. | select(has("creationTimestamp")).creationTimestamp) |\n\t\t\tdel(.. | select(has("namespace")).namespace) |\n\t\t\tdel(.. | select(has("uid")).uid) |\n\t\t\tdel(.metadata.resourceVersion) |\n\t\t\tdel(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |\n\t\t\tdel(.metadata.selfLink) |\n\t\t\tdel(.metadata.annotations."cloud.google.com/neg") |\n\t\t\tdel(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |\n\t\t\tdel(.. | select(has("image")).image) |\n\t\t\tdel(.. | select(has("clusterIP")).clusterIP) |\n\t\t\tdel(.. | select(has("clusterIPs")).clusterIPs) |\n\t\t\tdel(.. | select(has("dataSource")).dataSource) |\n\t\t\tdel(.. | select(has("procMount")).procMount) |\n\t\t\tdel(.. | select(has("storageClassName")).storageClassName) |\n\t\t\tdel(.. | select(has("finalizers")).finalizers) |\n\t\t\tdel(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |\n\t\t\tdel(.. | select(has("volumeName")).volumeName) |\n\t\t\tdel(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |\n\t\t\tdel(.spec.volumeMode) |\n\t\t\tdel(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |\n\t\t\tdel(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |\n\t\t\tdel(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |\n\t\t\tdel(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |\n\t\t\tdel(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |\n\t\t\tdel(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |\n\t\t\tdel(.. | select(has("nodePort")).nodePort) |\n\t\t\tdel(.status) |\n\t\t\t(.. | select(tag == "!!str")) |= sub("upgrade-consistency-sharded-tls-8065", "NAME_SPACE") |\n\t\t\tdel(.spec.volumeClaimTemplates[].apiVersion) |\n\t\t\tdel(.spec.volumeClaimTemplates[].kind) |\n\t\t\tdel(.spec.ipFamilies) |\n\t\t\tdel(.spec.ipFamilyPolicy) |\n\t\t\t(.. | select(. == "extensions/v1beta1")) = "apps/v1" |\n\t\t\t(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.KaPjLpZoRR
++ mktemp
+ local LAST_ERR=/tmp/tmp.wvFAQ6fIum
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KaPjLpZoRR
+ cat /tmp/tmp.wvFAQ6fIum
+ rm /tmp/tmp.KaPjLpZoRR /tmp/tmp.wvFAQ6fIum
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1230.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1230.yml /tmp/tmp.btY8OyXru1/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2026-04-12T21:59:29+0000] compare_kubectl: statefulset/some-name-cfg OK
+ destroy upgrade-consistency-sharded-tls-8065
+ local namespace=upgrade-consistency-sharded-tls-8065
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JIWgQNn34B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nPS8jGsEYE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JIWgQNn34B
++ cat /tmp/tmp.nPS8jGsEYE
No resources found in upgrade-consistency-sharded-tls-8065 namespace.
++ rm /tmp/tmp.JIWgQNn34B /tmp/tmp.nPS8jGsEYE
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.ISQQxLcvYc
++ mktemp
+ local LAST_ERR=/tmp/tmp.omgGYmMgBx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ISQQxLcvYc
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.omgGYmMgBx
+ rm /tmp/tmp.ISQQxLcvYc /tmp/tmp.omgGYmMgBx
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.6olbu2bEMh
++ mktemp
+ local LAST_ERR=/tmp/tmp.LlCmTdQlSy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6olbu2bEMh
+ cat /tmp/tmp.LlCmTdQlSy
+ rm /tmp/tmp.6olbu2bEMh /tmp/tmp.LlCmTdQlSy
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.cEuynZnr6h
++ mktemp
+ local LAST_ERR=/tmp/tmp.BXOnnb0QmQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cEuynZnr6h
+ cat /tmp/tmp.BXOnnb0QmQ
+ rm /tmp/tmp.cEuynZnr6h /tmp/tmp.BXOnnb0QmQ
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n upgrade-consistency-sharded-tls-8065 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.aAYDRpSvdv
++ mktemp
+ local LAST_ERR=/tmp/tmp.aOfRdKEpRn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aAYDRpSvdv
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.aOfRdKEpRn
+ rm /tmp/tmp.aAYDRpSvdv /tmp/tmp.aOfRdKEpRn
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.mgv9NIHpHY
++ mktemp
+ local LAST_ERR=/tmp/tmp.34DVQAl5au
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2266/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mgv9NIHpHY
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.34DVQAl5au
+ rm /tmp/tmp.mgv9NIHpHY /tmp/tmp.34DVQAl5au
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PlHzNdvsgh
++ mktemp
+ local LAST_ERR=/tmp/tmp.PCBwSKu0yI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PlHzNdvsgh
namespace "cert-manager" deleted
customresourcedefinition.apiextensions.k8s.io "challenges.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "orders.acme.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificaterequests.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "certificates.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "clusterissuers.cert-manager.io" deleted
customresourcedefinition.apiextensions.k8s.io "issuers.cert-manager.io" deleted
serviceaccount "cert-manager-cainjector" deleted from cert-manager namespace
serviceaccount "cert-manager" deleted from cert-manager namespace
serviceaccount "cert-manager-webhook" deleted from cert-manager namespace
clusterrole.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-cluster-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-view" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-edit" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrole.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-cainjector" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-issuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificates" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-orders" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-challenges" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" deleted
clusterrolebinding.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" deleted
role.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
role.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
role.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager:leaderelection" deleted from kube-system namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-tokenrequest" deleted from cert-manager namespace
rolebinding.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" deleted from cert-manager namespace
deployment.apps "cert-manager-cainjector" deleted from cert-manager namespace
deployment.apps "cert-manager" deleted from cert-manager namespace
deployment.apps "cert-manager-webhook" deleted from cert-manager namespace
mutatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
validatingwebhookconfiguration.admissionregistration.k8s.io "cert-manager-webhook" deleted
+ cat /tmp/tmp.PCBwSKu0yI
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PlHzNdvsgh
namespace "cert-manager" deleted
+ cat /tmp/tmp.PCBwSKu0yI
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PlHzNdvsgh
+ cat /tmp/tmp.PCBwSKu0yI
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.PlHzNdvsgh
+ cat /tmp/tmp.PCBwSKu0yI
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.PlHzNdvsgh /tmp/tmp.PCBwSKu0yI
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace upgrade-consistency-sharded-tls-8065
+ rm -rf /tmp/tmp.btY8OyXru1
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.KwNIhfpyX5
+ local LAST_OUT=/tmp/tmp.dZ70ehPFDz
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.UICuVi7UnB
+ local LAST_ERR=/tmp/tmp.DUHoBWdbXz
+ local exit_status=0
+ local exit_status=0
+ local timeout=4
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ kubectl delete --grace-period=0 --force=true namespace upgrade-consistency-sharded-tls-8065